Merge remote-tracking branch 'origin/master'

lib/apidigikey.py

+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+import traceback
+
+import redis
+import requests
+from retry import retry
+import token_lib
+from utils.robots import dd_send_msg
+from log_manage import get_logger
+
+apidigikey_log = get_logger('apidigikey_token_info')
+PLATFORM = "apidigikey"
+LEVEL = "C"
+BACK_NUM = 0
+MODE = "redis"
+MAIN_NUM = THREAD_NUM = 1
+
+p_headers = {
+    'Host': 'api.digikey.com',
+    'Content-Type': 'application/x-www-form-urlencoded'
+}
+p_url = "https://api.digikey.com/v1/oauth2/token"
+redirect_uri = 'https://bigdata.ichunt.com/digikey_auth.php'
+# client_id = '9zwA2tsTFpKURpMIj5VXGK3eoPWLIGgh'
+# client_secret = 'CxRLozO8hZ2GvCJ1'
+
+# 对接digikey的api的token不放在master而放在这台机器上
+apidigikey_ip = "172.18.137.38"
+psd = 'icDb29mLy2s'
+
+
+class DGTokenCrawler(token_lib.BaseTokenCrawler):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.token_key = "digikey_access_token"
+        self.refresh_token_key = "digikey_refresh_token"
+        self.client_id = '9zwA2tsTFpKURpMIj5VXGK3eoPWLIGgh'
+        self.client_secret = 'CxRLozO8hZ2GvCJ1'
+        self.redirect_uri = 'https://bigdata.ichunt.com/digikey_auth.php'
+        self._pool = redis.ConnectionPool(host=apidigikey_ip, port=6379, db=0, password=psd)
+        self._redis = redis.Redis(connection_pool=self._pool)
+
+    def get_params_list(self):
+        return [[i] for i in range(THREAD_NUM)]
+
+    def collect_token(self):
+        try:
+            access_token, refresh_token = self.get_token_by_refresh_token()
+            apidigikey_log.info(access_token)
+            apidigikey_log.info(refresh_token)
+            return access_token, refresh_token
+        except Exception:
+            dd_send_msg('digikey token同步失败：' + traceback.format_exc())
+            self.logger.error(traceback.format_exc())
+
+    def get_token(self):
+
+        # url="https://api.digikey.com/v1/oauth2/authorize?response_type=code&client_id=9zwA2tsTFpKURpMIj5VXGK3eoPWLIGgh&redirect_uri=https://bigdata.ichunt.com/digikey_auth.php"
+        # url="https://sandbox-api.digikey.com/v1/oauth2/authorize?client_id=Qe3ffZGQ9E5fTsOFiz4Cv69IzlFGpzvj&response_type=code&redirect_uri=https://bigdata.ichunt.com/digikey_auth_sandbox.php"
+        p_data = {
+            'code': 'XgjehVDz',
+            'client_id': self.client_id,
+            'client_secret': self.client_secret,
+            'redirect_uri': self.redirect_uri,
+            'grant_type': 'authorization_code'
+        }
+        # sess.max_redirects=0
+        resp = requests.post(p_url, headers=p_headers, data=p_data)
+        token_json = resp.json()
+        return token_json['access_token'], token_json['refresh_token']
+
+    @retry(tries=3)
+    def get_token_by_refresh_token(self):
+        refresh_token = self._redis.get(self.refresh_token_key).decode()
+        p_data = {
+            'client_id': self.client_id,
+            'client_secret': self.client_secret,
+            'refresh_token': refresh_token,
+            # 'refresh_token': 'jwki255qAu14VPDGhuJKQVPHixiAGO1D',
+            'grant_type': 'refresh_token'
+        }
+        resp = requests.post(p_url, headers=p_headers, data=p_data, timeout=(15, 20))
+        token_json = resp.json()
+        return token_json['access_token'], token_json['refresh_token']
+
+    def process(self, is_main):
+        access_token, refresh_token = self.collect_token()
+
+        self._redis.set(self.token_key, access_token)
+        self._redis.set(self.refresh_token_key, refresh_token)

token_server.py

@@ -13,7 +13,8 @@ import lib
 import token_lib
 from log_manage import get_logger
 
-process_list = ()
+# 去掉digikey的爬虫token
+process_list = ['digikey']
 logger = get_logger('app')
 
 
@@ -22,6 +23,7 @@ LEVEL_MAP = {
     "B": 30,  # 仅供测试用
     "X": 60,
     "C": 60 * 5,
+    "TM": 60 * 20,
     "HH": 60 * 30,
     "H": 60 * 60,
     "4H": 60 * 60 * 4,

utils/__init__.py

+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
\ No newline at end of file

utils/robots.py

+#!/usr/bin/env python
+# -*- coding:utf-8 -*-
+import time
+
+import requests
+
+dd_robot_api = ' https://oapi.dingtalk.com/robot/send?access_token='
+tom = 'c19beeab837f5c6e019e7fc602d470704d634ae33510f40fb0e903cde215fb24'
+
+
+def dd_send_msg(msg, robot=tom):
+    data = {
+        "msgtype": "text",
+        "text": {
+            "content": msg
+        }
+    }
+    requests.post(dd_robot_api + robot, json=data)
+    time.sleep(5)
+
+
+if __name__ == "__main__":
+    dd_send_msg('喂喂 你发的太多了！')