数据系统业务数据汇总表逻辑、用户数据汇总表逻辑

AdminController/dataSystemManagement/BusinessDataController.py

@@ -8,10 +8,10 @@
 """
 
 import requests
-from django.db.models import Q, Count, Sum
+from django.db.models import Count, Sum
 from django.views.generic.base import View
 
-from Model.models import VodHlsModel, VideoPlaybackTimeModel, VodHlsSummary
+from Model.models import VodHlsModel, VodHlsSummary
 from Service.CommonService import CommonService
 
 
@@ -54,49 +54,30 @@ class BusinessDataView(View):
         if not all([start_time, end_time]):
             return response.json(444, {'error param': 'startTime or endTime'})
         try:
-            vod_hls_summary_qs = VodHlsSummary.objects.filter(time__range=(start_time, end_time)).values()
-            vod_hls_qs = VodHlsModel.objects.filter(time__range=(start_time, end_time))
-            video_play_qs = VideoPlaybackTimeModel.objects.filter(startTime__range=(start_time, end_time),
-                                                                  playMode='could')
-            upload_duration_qs = vod_hls_qs.values('uid').annotate(uploadFrequency=Count('uid')).order_by(
-                'uploadFrequency')
-            uid_list = []
-            for item in upload_duration_qs:
-                item['uploadDuration'] = vod_hls_qs.filter(uid=item['uid']).aggregate(total=Sum('sec'))['total']
-            upload_duration_list = list(upload_duration_qs)
+            vod_hls_summary_qs = VodHlsSummary.objects.filter(time__gte=start_time, time__lt=end_time).values()
+            vod_hls_qs = VodHlsModel.objects.filter(endTime__gte=start_time, endTime__lt=end_time).values(
+                'uid').annotate(uploadFrequency=Count('uid'), uploadDuration=Sum('sec')).order_by('uploadFrequency')
+            vod_hls_list = list(vod_hls_qs)
             for each in vod_hls_summary_qs:
                 flag = 0
-                for item in upload_duration_list:
+                for item in vod_hls_list:
                     if each['uid'] == item['uid']:
-                        item['uploadDuration'] += each['sec']
+                        item['uploadDuration'] += each['upload_duration']
                         item['uploadFrequency'] += each['upload_frequency']
                         flag = 1
                 if flag == 0:
-                    upload_duration_list.append({
+                    vod_hls_list.append({
                         'uid': each['uid'],
-                        'uploadDuration': each['sec'],
-                        'uploadFrequency': each['upload_frequency']
+                        'uploadDuration': each['upload_duration'],
+                        'uploadFrequency': each['upload_frequency'],
+                        'playFrequency': 0,
+                        'playDuration': 0
                     })
-            for item in upload_duration_list:
-                item['playDuration'] = video_play_qs.filter(uid=item['uid']).aggregate(total=Sum('duration'))['total']
-                item['playDuration'] = item['playDuration'] if item['playDuration'] else 0
-                item['playFrequency'] = video_play_qs.filter(uid=item['uid'], playMode='cloud').count()
-                uid_list.append(item['uid'])
+            upload_device_count = len(vod_hls_list)  # 上传设备数量
 
-            upload_device_count = len(upload_duration_list)  # 上传设备数量
-            video_play_qs = video_play_qs.filter(~Q(uid__in=uid_list))
-            play_duration_qs = video_play_qs.values('uid').annotate(playFrequency=Count('uid')).order_by(
-                'playFrequency')
-            play_device_count = play_duration_qs.count()
-            for item in play_duration_qs:
-                item['uploadFrequency'] = 0
-                item['uploadDuration'] = 0
-                item['playDuration'] = video_play_qs.filter(uid=item['uid']).aggregate(total=Sum('duration'))['total']
-                item['playDuration'] = item['playDuration'] if item['playDuration'] else 0
             res = {
                 'uploadDeviceCount': upload_device_count,
-                'playDeviceCount': play_device_count,
-                'vodData': upload_duration_list + list(play_duration_qs)
+                'vodData': vod_hls_list
             }
             return response.json(0, res)
         except Exception as e:
@@ -117,7 +98,6 @@ class BusinessDataView(View):
                 'Authorization': request.META.get('HTTP_AUTHORIZATION')
             }
             upload_device_count = 0
-            play_device_count = 0
             vod_list = []
             for url in url_list:
                 url = url + request.path.replace('global/', '')
@@ -125,13 +105,11 @@ class BusinessDataView(View):
                 result = res.json()
                 if result['result_code'] == 0:
                     upload_device_count += int(result['result']['uploadDeviceCount'])
-                    play_device_count += int(result['result']['playDeviceCount'])
                     vod_list += result['result']['vodData']
                 else:
                     return response.json(result['result_code'], result['result'])
             res = {
                 'uploadDeviceCount': upload_device_count,
-                'playDeviceCount': play_device_count,
                 'vodData': vod_list
             }
             return response.json(0, res)

AdminController/dataSystemManagement/UserDataController.py

@@ -7,12 +7,12 @@
 @Software: PyCharm
 """
 
-from django.db.models import Count
+from django.db.models import Count, Sum
 from django.views.generic.base import View
 import datetime
 import requests
 
-from Model.models import Device_User, CountryModel
+from Model.models import Device_User, CountryModel, DeviceUserSummary
 from Service.CommonService import CommonService
 
 
@@ -64,45 +64,46 @@ class UserDataView(View):
         time_unit = request_dict.get('timeUnit', None)
         if not all([start_time, end_time, time_unit]):
             return response.json(444, {'error param': 'startTime or endTime or timeUnit'})
-        start_time = datetime.datetime.fromtimestamp(int(start_time))
-        end_time = datetime.datetime.fromtimestamp(int(end_time))
-        time_list = CommonService.cutting_time(start_time, end_time, time_unit)
         try:
-            user_qs = Device_User.objects.filter(data_joined__range=(start_time, end_time))
+            all_user_qs = DeviceUserSummary.objects.filter(time__gte=start_time, time__lt=end_time,
+                                                           query_type=0).values('count', 'country')
+            count = all_user_qs.aggregate(total=Sum('count'))['total']
+            # 处理用户地区
+            region_list = []
+            for item in all_user_qs:
+                country_temp_dict = eval(item['country'])
+                for k, v in country_temp_dict.items():
+                    flag = 0
+                    for each in region_list:
+                        if k == each['countryName']:
+                            each['count'] += v
+                            flag = 1
+                            break
+                    if flag == 0:
+                        region_list.append({
+                            'countryName': k,
+                            'count': v
+                        })
+            for item in region_list:
+                item['rate'] = round(item['count'] / count * 100, 2) if count else 0
+            # 处理时间段用户
+            start_time = datetime.datetime.fromtimestamp(int(start_time))
+            end_time = datetime.datetime.fromtimestamp(int(end_time))
+            time_list = CommonService.cutting_time(start_time, end_time, time_unit)
             user_list = []
-            count = user_qs.count()
             for item in time_list:
-                s_time = datetime.datetime.fromtimestamp(int(item[0]))
-                e_time = datetime.datetime.fromtimestamp(int(item[1]))
-                user_date_qs = user_qs.filter(last_login__range=(s_time, e_time))
-                if count:
-                    rate = round(user_date_qs.count() / count * 100, 2)
-                else:
-                    rate = 0
+                user_qs = all_user_qs.filter(time__gte=item[0], time__lt=item[1], query_type=0).values('count')
+                temp_count = user_qs.aggregate(total=Sum('count'))['total']
+                temp_count = temp_count if temp_count else 0
+                rate = round(temp_count / count * 100, 2) if count else 0
                 user_dict = {
-                    'count': user_date_qs.count(),
+                    'count': temp_count,
                     'rate': rate,
                     'startTime': item[0],
                     'endTime': item[1]
                 }
                 user_list.append(user_dict)
-            user_country_qs = user_qs.values('region_country').annotate(count=Count('region_country')).order_by(
-                '-count')
-            region_list = []
-            for item in user_country_qs:
-                country_id = item['region_country']
-                country_qs = CountryModel.objects.filter(id=country_id).values('country_name')
-                country_name = country_qs[0]['country_name'] if country_qs.exists() else '未知区域'
-                if count:
-                    rate = round(item['count'] / count * 100, 2)
-                else:
-                    rate = 0
-                country_dict = {
-                    'countryName': country_name,
-                    'count': item['count'],
-                    'rate': rate
-                }
-                region_list.append(country_dict)
+
             res = {
                 'user': user_list,
                 'region': region_list
@@ -131,7 +132,7 @@ class UserDataView(View):
         end_time = datetime.datetime.fromtimestamp(int(end_time))
         time_list = CommonService.cutting_time(start_time, end_time, time_unit)
         try:
-            user_qs = Device_User.objects.filter(last_login__range=(start_time, end_time))
+            user_qs = Device_User.objects.filter(last_login__range=(start_time, end_time)).values('region_country', )
             user_list = []
             count = user_qs.count()
             for item in time_list:
@@ -152,7 +153,7 @@ class UserDataView(View):
             user_country_qs = user_qs.values('region_country').annotate(count=Count('region_country')).order_by(
                 '-count')
             region_list = []
-            for item in user_country_qs:
+            for item in user_country_qs[:30]:
                 country_id = item['region_country']
                 country_qs = CountryModel.objects.filter(id=country_id).values('country_name')
                 country_name = country_qs[0]['country_name'] if country_qs.exists() else '未知区域'

Ansjer/urls.py

@@ -396,6 +396,8 @@ urlpatterns = [
     re_path(r'^cron/del/(?P<operation>.*)', CronTaskController.CronDelDataView.as_view()),
     # 定时更新任务接口
     re_path(r'^cron/update/(?P<operation>.*)', CronTaskController.CronUpdateDataView.as_view()),
+    # 定时收集数据任务接口
+    re_path(r'^cron/collect/(?P<operation>.*)', CronTaskController.CronCollectDataView.as_view()),
 
     re_path('(?P<path>.*)', LogManager.errorPath),
 

Controller/Cron/CronTaskController.py

@@ -11,11 +11,12 @@ import datetime
 import time
 
 from django.db import connection, connections, transaction
-from django.db.models import Q
+from django.db.models import Q, Sum, Count
 from django.views import View
 
 from Model.models import Device_User, Device_Info, UidSetModel, UID_Bucket, Unused_Uid_Meal, Order_Model, StsCrdModel, \
-    VodHlsModel, ExperienceContextModel, AiService, VodHlsSummary
+    VodHlsModel, ExperienceContextModel, AiService, VodHlsSummary, VideoPlaybackTimeModel, DeviceUserSummary, \
+    CountryModel
 from Object.ResponseObject import ResponseObject
 from Object.utils import LocalDateTimeUtil
 from Service.CommonService import CommonService
@@ -104,10 +105,9 @@ class CronDelDataView(View):
     @staticmethod
     def delVodHls(response):
         nowTime = int(time.time())
-        # cursor = connection.cursor()
         try:
             with transaction.atomic():
-                month_ago_time = nowTime - 1 * 30 * 24 * 60 * 60
+                month_ago_time = nowTime - 30 * 24 * 60 * 60  # 删除1个月前的数据
                 vod_hls_qs = VodHlsModel.objects.filter(endTime__lte=month_ago_time)
                 for vod_hls in vod_hls_qs:
                     end_time = vod_hls.endTime
@@ -115,21 +115,17 @@ class CronDelDataView(View):
                     this_month_start = datetime.datetime(end_time_str.year, end_time_str.month, 1)
                     this_month_start_stamp = CommonService.str_to_timestamp(
                         this_month_start.strftime('%Y-%m-%d %H:%M:%S'))
-                    vod_business_qs = VodHlsSummary.objects.filter(time=this_month_start_stamp, uid=vod_hls.uid)
-                    if vod_business_qs.exists():
-                        vod_business = vod_business_qs.first()
-                        vod_business.sec = vod_business.sec + vod_hls.sec
-                        vod_business.upload_frequency = vod_business.upload_frequency + 1
-                        vod_business.save()
+                    vod_hls_summary_qs = VodHlsSummary.objects.filter(time=this_month_start_stamp,
+                                                                      uid=vod_hls.uid)
+                    if vod_hls_summary_qs.exists():
+                        vod_hls_summary = vod_hls_summary_qs.first()
+                        vod_hls_summary.upload_duration = vod_hls_summary.upload_duration + vod_hls.sec
+                        vod_hls_summary.upload_frequency = vod_hls_summary.upload_frequency + 1
+                        vod_hls_summary.save()
                     else:
-                        VodHlsSummary.objects.create(time=this_month_start_stamp, uid=vod_hls.uid, sec=vod_hls.sec)
+                        VodHlsSummary.objects.create(time=this_month_start_stamp, uid=vod_hls.uid,
+                                                     upload_duration=vod_hls.sec, upload_frequency=1)
                     vod_hls.delete()
-                # # 删除1个月前的数据
-                # sql = "DELETE FROM `vod_hls` WHERE endTime<={} LIMIT 50000".format(
-                #     nowTime - 1 * 30 * 24 * 60 * 60)
-                # cursor.execute(sql)
-                # 关闭游标
-            # cursor.close()
             return response.json(0)
         except Exception as e:
             return response.json(500, repr(e))
@@ -294,3 +290,129 @@ class CronUpdateDataView(View):
             except Exception:
                 continue
         return response.json(0)
+
+
+class CronCollectDataView(View):
+    def get(self, request, *args, **kwargs):
+        request.encoding = 'utf-8'
+        operation = kwargs.get('operation')
+        return self.validation(request.GET, request, operation)
+
+    def post(self, request, *args, **kwargs):
+        request.encoding = 'utf-8'
+        operation = kwargs.get('operation')
+        return self.validation(request.POST, request, operation)
+
+    def validation(self, request_dict, request, operation):
+        response = ResponseObject()
+        if operation == 'collectPlayBack':  # 定时保存云存视频回放
+            return self.collect_play_back(response)
+        if operation == 'collectDeviceUser':  # 定时保存云存视频回放
+            return self.collect_device_user(response)
+        else:
+            return response.json(404)
+
+    @staticmethod
+    def collect_play_back(response):
+        try:
+            end_time = int(time.time())
+            start_time = end_time - 24 * 60 * 60  # 每天执行一次
+            today = datetime.datetime.today()
+            this_month_str = datetime.datetime(today.year, today.month, 1)
+            this_month_stamp = CommonService.str_to_timestamp(this_month_str.strftime('%Y-%m-%d %H:%M:%S'))
+            video_play_back_time_qs = VideoPlaybackTimeModel.objects.filter(startTime__gte=start_time,
+                                                                            startTime__lt=end_time,
+                                                                            playMode='cloud').values('uid').annotate(
+                play_duration=Sum('duration'), play_frequency=Count('uid'))
+            with transaction.atomic():
+                for item in video_play_back_time_qs:
+                    vod_hls_summary_qs = VodHlsSummary.objects.filter(uid=item['uid'], time=this_month_stamp)
+                    if vod_hls_summary_qs.exists():
+                        vod_hls_summary = vod_hls_summary_qs.first()
+                        vod_hls_summary.play_duration += item['play_duration']
+                        vod_hls_summary.play_frequency += 1
+                        vod_hls_summary.save()
+                    else:
+                        VodHlsSummary.objects.create(uid=item['uid'], time=this_month_stamp,
+                                                     play_duration=item['play_duration'], play_frequency=1)
+
+            return response.json(0)
+        except Exception as e:
+            return response.json(500, repr(e))
+
+    @staticmethod
+    def collect_device_user(response):
+        try:
+            today = datetime.datetime.today()
+            start_time = datetime.datetime(today.year, today.month, today.day)
+            end_time = start_time + datetime.timedelta(days=1)
+            increase_user_qs = Device_User.objects.filter(data_joined__gte=start_time, data_joined__lt=end_time).values(
+                'data_joined',
+                'region_country')
+            active_user_qs = Device_User.objects.filter(last_login__gte=start_time, last_login__lt=end_time).values(
+                'last_login',
+                'region_country')
+            start_time = CommonService.str_to_timestamp(start_time.strftime('%Y-%m-%d %H:%M:%S'))
+            country_qs = CountryModel.objects.all().values('id', 'region__name', 'country_name')
+            country_dict = {}
+            continent_dict = {}
+            for item in country_qs:
+                country_dict[item['id']] = item['country_name']
+                continent_dict[item['country_name']] = item['region__name']
+            with transaction.atomic():
+                for item in increase_user_qs:
+                    device_user_summary_qs = DeviceUserSummary.objects.filter(time=start_time, query_type=0)
+                    country_name = country_dict.get(item['region_country'], '未知国家')
+                    continent_name = continent_dict.get(country_name, '未知大洲')
+                    if device_user_summary_qs.exists():
+                        device_user_summary = device_user_summary_qs.first()
+                        country_temp_dict = eval(device_user_summary.country)
+                        continent_temp_dict = eval(device_user_summary.continent)
+                        if country_name in country_temp_dict:
+                            country_temp_dict[country_name] += 1
+                        else:
+                            country_temp_dict[country_name] = 1
+                        if continent_name in continent_temp_dict:
+                            continent_temp_dict[continent_name] += 1
+                        else:
+                            continent_temp_dict[continent_name] = 1
+                        device_user_summary.country = country_temp_dict
+                        device_user_summary.continent = continent_temp_dict
+                        device_user_summary.count += 1
+                        device_user_summary.save()
+                    else:
+                        country_temp_dict = {country_name: 1}
+                        continent_temp_dict = {continent_name: 1}
+                        DeviceUserSummary.objects.create(time=start_time, count=1,
+                                                         country=country_temp_dict,
+                                                         continent=continent_temp_dict)
+                for item in active_user_qs:
+                    device_user_summary_qs = DeviceUserSummary.objects.filter(time=start_time, query_type=1)
+                    country_name = country_dict.get(item['region_country'], '未知国家')
+                    continent_name = continent_dict.get(country_name, '未知大洲')
+                    if device_user_summary_qs.exists():
+                        device_user_summary = device_user_summary_qs.first()
+                        country_temp_dict = eval(device_user_summary.country)
+                        continent_temp_dict = eval(device_user_summary.continent)
+                        if country_name in country_temp_dict:
+                            country_temp_dict[country_name] += 1
+                        else:
+                            country_temp_dict[country_name] = 1
+                        if continent_name in continent_temp_dict:
+                            continent_temp_dict[continent_name] += 1
+                        else:
+                            continent_temp_dict[continent_name] = 1
+                        device_user_summary.country = country_temp_dict
+                        device_user_summary.continent = continent_temp_dict
+                        device_user_summary.count += 1
+                        device_user_summary.save()
+                    else:
+                        country_temp_dict = {country_name: 1}
+                        continent_temp_dict = {continent_name: 1}
+                        DeviceUserSummary.objects.create(time=start_time, query_type=1, count=1,
+                                                         country=country_temp_dict,
+                                                         continent=continent_temp_dict)
+
+            return response.json(0)
+        except Exception as e:
+            return response.json(500, repr(e))

Model/models.py

@@ -171,8 +171,8 @@ class Device_User(AbstractBaseUser):
     NickName = models.CharField(blank=True, max_length=64, default='', verbose_name=u'用户昵称')
     is_superuser = models.IntegerField(blank=True, default=0, verbose_name=u'用户类型')
     is_active = models.BooleanField(blank=True, default=False, verbose_name=u'用户活动状态')
-    data_joined = models.DateTimeField(blank=True, verbose_name=u'加入时间', auto_now_add=True)
-    last_login = models.DateTimeField(blank=True, verbose_name='u最后登录时间', auto_now=True)
+    data_joined = models.DateTimeField(blank=True, verbose_name=u'加入时间', auto_now_add=True, db_index=True)
+    last_login = models.DateTimeField(blank=True, verbose_name='u最后登录时间', auto_now=True, db_index=True)
     user_isValid = models.BooleanField(blank=True, default=False, verbose_name=u'用户有效性', help_text=u'是否是认证通过的用户')
     online = models.BooleanField(blank=True, default=False, verbose_name=u'用户登录状态')
     machine_code = models.CharField(blank=True, max_length=128, default='', verbose_name=u'机械码')
@@ -265,7 +265,7 @@ class Device_Info(models.Model):
     primaryMaster = models.CharField(max_length=64, verbose_name=u'主用户名', default='')
     vodPrimaryUserID = models.CharField(blank=True, verbose_name='云存主用户id', max_length=32, default='')
     vodPrimaryMaster = models.CharField(max_length=64, verbose_name=u'云存主用户名', default='')
-    data_joined = models.DateTimeField(blank=True, verbose_name=u'设备添加时间', auto_now_add=True)
+    data_joined = models.DateTimeField(blank=True, verbose_name=u'设备添加时间', auto_now_add=True, db_index=True)
     update_time = models.DateTimeField(blank=True, verbose_name=u'更新时间', auto_now=True, null=True)
 
     ip = models.CharField(blank=True, max_length=20, default='', verbose_name=u'设备区域ip')
@@ -1037,23 +1037,6 @@ class VodHlsModel(models.Model):
         ordering = ('-id',)
 
 
-class VodHlsSummary(models.Model):
-    id = models.AutoField(primary_key=True, verbose_name='回放汇总列表主键')
-    uid = models.CharField(max_length=20, verbose_name='设备UID', db_index=True)
-    time = models.IntegerField(verbose_name='汇总年月时间戳', default=0, db_index=True)
-    sec = models.IntegerField(verbose_name='上传秒数', default=0)
-    upload_frequency = models.SmallIntegerField(verbose_name='上传次数', default=1)
-
-    def __str__(self):
-        return self.id
-
-    class Meta:
-        db_table = 'vod_hls_summary'
-        verbose_name = u'云存回放汇总信息表'
-        verbose_name_plural = verbose_name
-        ordering = ('-id',)
-
-
 class OssCrdModel(models.Model):
     id = models.AutoField(primary_key=True, verbose_name='主键')
     uid = models.CharField(max_length=20, verbose_name='设备UID')
@@ -1172,8 +1155,8 @@ class UidSetModel(models.Model):
     channel = models.SmallIntegerField(default=0, verbose_name='通道数量')  #
     detect_status = models.SmallIntegerField(default=0, verbose_name='推送开关')  # 状态[0:关闭,1:开启,2:用户解绑]
     detect_interval = models.IntegerField(verbose_name='推送间隔', default=60)  # 秒
-    addTime = models.IntegerField(verbose_name='添加时间', default=0)
-    updTime = models.IntegerField(verbose_name='更新时间', default=0)
+    addTime = models.IntegerField(verbose_name='添加时间', default=0, db_index=True)
+    updTime = models.IntegerField(verbose_name='更新时间', default=0, db_index=True)
     ucode = models.CharField(max_length=32, verbose_name='设备产品码', default='')
     version = models.CharField(max_length=32, verbose_name='设备版本', default='')
     p2p_region = models.CharField(max_length=16, verbose_name='设备p2p区域', default='ALL')  # ALL CN EU US
@@ -2905,3 +2888,38 @@ class DeviceUidAlgorithmType(models.Model):
         db_table = 'device_uid_algorithm_type'
         verbose_name = '设备uid关联算法类型'
         verbose_name_plural = verbose_name
+
+
+class VodHlsSummary(models.Model):
+    id = models.AutoField(primary_key=True, verbose_name='回放汇总列表主键')
+    uid = models.CharField(max_length=20, verbose_name='设备UID', db_index=True)
+    time = models.IntegerField(verbose_name='汇总年月时间戳', default=0, db_index=True)
+    upload_duration = models.IntegerField(verbose_name='上传秒数', default=0)
+    upload_frequency = models.SmallIntegerField(verbose_name='上传次数', default=0)
+    play_duration = models.IntegerField(verbose_name='播放秒数', default=0)
+    play_frequency = models.SmallIntegerField(verbose_name='播放次数', default=0)
+
+    def __str__(self):
+        return self.id
+
+    class Meta:
+        db_table = 'vod_hls_summary'
+        verbose_name = u'云存回放汇总信息表'
+        verbose_name_plural = verbose_name
+
+
+class DeviceUserSummary(models.Model):
+    id = models.AutoField(primary_key=True, verbose_name='用户汇总列表主键')
+    time = models.IntegerField(verbose_name='时间戳', default=0, db_index=True)
+    query_type = models.SmallIntegerField(verbose_name='查询类型{0：新增用户，1：活跃用户}', default=0)
+    count = models.IntegerField(verbose_name='总人数', default=0)
+    country = models.CharField(verbose_name='国家信息', max_length=255, default='')
+    continent = models.CharField(verbose_name='四大洲信息', max_length=255, default='')
+
+    def __str__(self):
+        return self.id
+
+    class Meta:
+        db_table = 'device_user_summary'
+        verbose_name = u'用户汇总信息表'
+        verbose_name_plural = verbose_name