优化品牌统计接口

Controller/UserBrandController.py

@@ -256,16 +256,39 @@ class MyserviceDynamodb(object):
 
         response = table.scan()
         return len(response['Items'])
-
+# 已经查询全部
     def item_get_brand(self, table_name):
         dynamodb = self.session.resource('dynamodb')
         table = dynamodb.Table(table_name)
-        try:
-            response = table.scan()
-            response = response['Items']
-            return response
-        except Exception:
-            logger.error("Failed to put item in to {0}:error{1}".format(table))
+        last_evaluated_key = None
+        i = 0
+        table_info = []
+        while True:
+            # 刚开始，不需要传入startkey
+            if last_evaluated_key is None:
+                response = table.scan()
+            else:
+                # 构建分页的起点，传入下一页面的起点，这是由主键来控制的，last_evaluated_key的值就是本表中的uid
+                try:
+                    response = table.scan(ExclusiveStartKey=last_evaluated_key)
+                except Exception:
+                    break
+            # print type(response) #字典
+            i = i + 1
+            print (i)
+            if response['Items'] != []:
+                table_info.extend(response['Items'])
+            # 判断有没有这个LastEvaluatedKey，如果有，那么还有页面没有拉取玩，将这个值带入request的ExclusiveStartKey中，继续读取页面
+            try:
+                if response['LastEvaluatedKey']:
+                    last_evaluated_key = response['LastEvaluatedKey']
+                    continue
+                else:
+                    break
+            except Exception:
+                break
+        return table_info
+
 
     def put_item(self, table, item_dict=None):
         try:
@@ -274,41 +297,6 @@ class MyserviceDynamodb(object):
             logger.error("Failed to put item in to {0}:error{1}".format(table, e))
         return response
 
-    def get_item(self, table_name, username):
-        dynamodb = self.session.resource('dynamodb')
-        # if not dynamodb:
-        #     raise DynamodbConnectionError("Failed to get resource for dynamodb!")
-        try:
-            table = dynamodb.Table(table_name)
-            response = table.scan(
-                FilterExpression=Attr('deviceSupplier').eq(username)
-                                 | Key('osType').eq(username)
-                                 | Key('deviceModel').eq(username)
-                                 | Key('userID').eq(username)
-                                 | Key('osVersion').eq(username)
-                                 | Key('country').eq(username)
-                                 | Key('appId').eq(username)
-                                 | Key('province').eq(username)
-                                 | Key('city').eq(username)
-                                 | Key('area').eq(username)
-                                 | Key('street').eq(username)
-                                 | Key('longitude').eq(username)
-                                 | Key('latitude').eq(username)
-                                 | Key('status_all').eq(username)
-                                 | Key('ip').eq(username)
-            )
-            items = response['Items']
-            for k, v in enumerate(items):
-                # print (items[k]['userID'])
-                user_ID = Device_User.objects.filter(userID=items[k]['userID']).values('userID', 'username', 'NickName')
-                try:
-                    items[k]['username'] = user_ID[0]['username']
-                except Exception:
-                    items[k]['username'] = ''
-        except Exception as e:
-            logger.error("Failed to get table {0}, error".format(table_name, e))
-        return items
-
     def update_table(self, table_name, user_id, username, add_time, deviceSupplier, deviceModel, osType, osVersion,
                      country, province, city, area, street, longitude, latitude, appId, ip, status_all
                      ):
@@ -374,6 +362,82 @@ class MyserviceDynamodb(object):
         except Exception:
             logger.error("Failed to put item in to {0}:error{1}".format(table))
             return 'no'
+# 搜索查询并分页查询全部
+    def get_table_info(self, table_name, username):
+        """
+        对表进行分页扫描，这里尝试的是对visit_info表进行扫描
+        """
+        # 通过dynamodb服务获取目标table的操作对象
+        dynamodb = self.session.resource('dynamodb')
+        table_handle = dynamodb.Table(table_name)
+        # 这个值是在分页查询的时候，用来记录页面的最后一个主键的下一个，以方便下一个页面的开启
+        last_evaluated_key = None
+        i = 0
+        table_info =[]
+        while True:
+            # 刚开始，不需要传入startkey
+            if last_evaluated_key is None:
+                response = table_handle.scan(
+                    FilterExpression=Attr('deviceSupplier').eq(username)
+                                 | Key('osType').eq(username)
+                                 | Key('deviceModel').eq(username)
+                                 | Key('userID').eq(username)
+                                 | Key('osVersion').eq(username)
+                                 | Key('country').eq(username)
+                                 | Key('appId').eq(username)
+                                 | Key('province').eq(username)
+                                 | Key('city').eq(username)
+                                 | Key('area').eq(username)
+                                 | Key('street').eq(username)
+                                 | Key('longitude').eq(username)
+                                 | Key('latitude').eq(username)
+                                 | Key('status_all').eq(username)
+                                 | Key('ip').eq(username))
+            else:
+                # 构建分页的起点，传入下一页面的起点，这是由主键来控制的，last_evaluated_key的值就是本表中的uid
+                try:
+                    response = table_handle.scan(
+                        ExclusiveStartKey=last_evaluated_key,
+                        FilterExpression=Attr('deviceSupplier').eq(username)
+                                     | Key('osType').eq(username)
+                                     | Key('deviceModel').eq(username)
+                                     | Key('userID').eq(username)
+                                     | Key('osVersion').eq(username)
+                                     | Key('country').eq(username)
+                                     | Key('appId').eq(username)
+                                     | Key('province').eq(username)
+                                     | Key('city').eq(username)
+                                     | Key('area').eq(username)
+                                     | Key('street').eq(username)
+                                     | Key('longitude').eq(username)
+                                     | Key('latitude').eq(username)
+                                     | Key('status_all').eq(username)
+                                     | Key('ip').eq(username))
+                except Exception:
+                    break
+            # response 有一个标准的json格式，包含了这次scan结果的各种信息
+            # print type(response) #字典
+            i = i+1
+            print (i)
+            if response['Items']!=[]:
+                table_info.extend(response['Items'])
+            # 判断有没有这个LastEvaluatedKey，如果有，那么还有页面没有拉取玩，将这个值带入request的ExclusiveStartKey中，继续读取页面
+            try:
+                if response['LastEvaluatedKey']:
+                    last_evaluated_key = response['LastEvaluatedKey']
+                    continue
+                else:
+                    break
+            except Exception:
+                break
+        for k, v in enumerate(table_info):
+            user_ID = Device_User.objects.filter(userID=table_info[k]['userID']).values('userID', 'username', 'NickName')
+            try:
+                table_info[k]['username'] = user_ID[0]['username']
+            except Exception:
+                table_info[k]['username'] = ''
+        return table_info
+        pass
 
 
 my = MyserviceDynamodb()
@@ -538,6 +602,7 @@ class UserBrandInfo(View):
 
     # 查询每个用户的最新登录数据表
     def query_info(self, request_dict, userID, user_brand, user_brand_all, response):
+
         page = int(request_dict.get('page', None))
         line = int(request_dict.get('line', None))
         username = request_dict.get('username', None)
@@ -565,8 +630,7 @@ class UserBrandInfo(View):
                         username = user_ID[0]['userID']
                     except Exception:
                         username = username
-                    table_value = my.get_item(user_brand, username)
-
+                    table_value = my.get_table_info(user_brand, username)
                     return response.json(0, {'datas': table_value, 'count': len(table_value)})
             else:
                 return response.json(404)
@@ -604,14 +668,6 @@ class UserBrandInfo(View):
             page_value = []
             if check_perm is True:
                 if username is None or username is '':
-                    # table_Limit_value = my.get_page_line(page, line, user_brand_all, None, page_value)
-                    # page = page - 1
-                    # if page < len(table_Limit_value):
-                    #     table_value = my.item_get(line, user_brand_all, table_Limit_value[page])
-                    #     table_value = table_value['Items']
-                    #     count = my.item_get_count(user_brand_all)
-                    #     return response.json(0, {'datas': table_value, 'count': count})
-
                     return response.json(0, {'datas': {}, 'count': 0})
                 else:
                     # 查询
@@ -620,8 +676,7 @@ class UserBrandInfo(View):
                         username = user_ID[0]['userID']
                     except Exception:
                         username = username
-
-                    table_value = my.get_item(user_brand_all, username)
+                    table_value = my.get_table_info(user_brand_all, username)
 
                     return response.json(0, {'datas': table_value, 'count': len(table_value)})
             else:

Service/middleware.py

@@ -77,7 +77,7 @@ class StatisticsUrlMiddleware(MiddlewareMixin):
         '''
         self._https_statistics_to_close(request)
         ########记录访问日志
-        MiscellService.DynamoDB_add_access_log(request=request, status_code=response.status_code)
+        # MiscellService.DynamoDB_add_access_log(request=request, status_code=response.status_code)
 
         if request.path !='/favicon.ico':
             print('process_response', request, response)