support oss domain

Author: lowkeyrd

publish.yaml

@@ -17,12 +17,10 @@ Parameters:
   type: object
   additionalProperties: false # 不允许增加其他属性
   required: # 必填项
-    - region
     - role
-    - functionName
-    - bucketName
-    - sourceUrl
-    - cron
+    - origin
+    - backupOrigin
+    - cronExpression
   properties:
     region:
       title: 地域
@@ -54,33 +52,50 @@ Parameters:
       type: string
       default: ''
       description: 函数执行时所扮演的角色
+      x-roleArn:
+        type: object
+        name: cdnbackuprole
+        pattern: '^acs:ram::[0-9]*:role/.*$'
+        service: fc
+        description: 应用所属的函数计算服务配置的 role, 请提前创建好对应的 role, 授信函数计算服务, 并配置好 AliyunOSSFullAccess 和 AliyunFCDefaultRolePolicy policy。
+        authorities:
+          - AliyunOSSFullAccess
+          - AliyunFCDefaultRolePolicy
     serviceName:
       title: 服务名
       type: string
-      default: cdn-appcenter-service
+      default: cdn-backup-origin
       description: 应用的服务名称
     functionName:
       title: 函数名
       type: string
-      default: cdn-backup-source
+      default: cdn-backup-origin
       description: 应用的函数名称
-    bucketName:
-      title: OSS Bucket名字
-      type: string
-      default: ''
-      description: '备源结果存储到指定的OSS Bucket'
-    sourceUrl:
+    origin:
       title: 源站地址
       type: string
-      default: ''
       description: 指定源站地址
-    warmupDomain:
+    backupOrigin:
       title: 备源地址
       type: string
       default: ''
-      description: 备源地址
-    cron:
+      description: '仅支持OSS Bucket域名, 例如: fc-layers.oss-cn-hangzhou.aliyuncs.com'
+    warmupDomain:
+      title: 预热域名
+      type: string
+      description: 抓取源站并预热到CDN节点
+    # maxFetchLevel:
+    #   title: 最大抓取层级
+    #   type: number
+    #   default: 0
+    #   description: 最大抓取层级
+    cronExpression:
       title: 定时触发时间
       type: string
       default: '0 0 0 ? * *'
       description: 指定定时触发时间
+    enable:
+      title: 是否启用任务
+      type: bool
+      default: true
+      description: 是否启用任务

src/code/index.py

@@ -1,13 +1,17 @@
 # -*- coding: utf-8 -*-
 
-import logging, os, re, time
-import urllib.parse, urllib.request
+import logging
+import os
+import re
+import time
+import urllib.parse
+import urllib.request
 from functools import reduce
 import oss2
 from aliyunsdkcore.client import AcsClient
 from aliyunsdkcore.request import CommonRequest
 from aliyunsdkcore.auth.credentials import AccessKeyCredential
-from aliyunsdkcore.auth.credentials import StsTokenCredential  
+from aliyunsdkcore.auth.credentials import StsTokenCredential
 
 LOGGER = logging.getLogger()
 
@@ -113,9 +117,16 @@ def download_file(src_url, dist_path):
 '''
 
 
-def parse_and_download_page(url):
+def parse_and_download_page(url, level):
     global downloaded_list
+    global max_level
 
+    # # 达到最大抓取层级，终止递归
+    if max_level > 0 and level > max_level:
+        LOGGER.info("max fetch level {0} reached".format(max_level))
+        return
+    
+    level += 1
     url_dict = parse_url(url)
     domain = url_dict['domain']
 
@@ -154,9 +165,11 @@ def parse_and_download_page(url):
         # http://www.baidu/com/static/index.js
         # static/js/index.js
         if resource_url.startswith('../'):
-            resource_url = urllib.parse.urljoin(url_dict['full_path'], resource_url)
+            resource_url = urllib.parse.urljoin(
+                url_dict['full_path'], resource_url)
         elif resource_url.startswith('./'):
-            resource_url = urllib.parse.urljoin(url_dict['full_path'], resource_url)
+            resource_url = urllib.parse.urljoin(
+                url_dict['full_path'], resource_url)
         elif resource_url.startswith('//'):
             resource_url = 'https:' + resource_url
         elif resource_url.startswith('/'):
@@ -189,7 +202,8 @@ def parse_and_download_page(url):
 
         # 递归解析 html/htm/shtml
         if resource_url_dict['ext'] in ['html', 'htm', 'shtml']:
-            parse_and_download_page(resource_url)
+            LOGGER.info("parse_and_download_page, level {}".format(level))
+            parse_and_download_page(resource_url, level)
 
         if not os.path.exists(resource_dir):
             os.makedirs(resource_dir)
@@ -219,11 +233,14 @@ def handler(event, context):
     global downloaded_list
     downloaded_list = []
 
+    global max_level
+    max_level = int(os.environ['MAX_FETCH_LEVEL'])
+
     # 备份源站（域名入口）
     # url = 'http://www.cgbchina.com.cn/index.html'
     # url = 'http://www.peersafe.cn/index.html'
-    url = os.environ['SOURCE_URL']
-    parse_and_download_page(url)
+    url = os.environ['ORIGIN']
+    parse_and_download_page(url, 0)
     LOGGER.info('总共下载了 %d 个资源' % len(downloaded_list))
 
     # 解析下载网页后备份和预热
@@ -234,37 +251,51 @@ def handler(event, context):
             creds.access_key_secret,
             creds.security_token)
         # 备份目标到OSS
-        bucket_name = os.environ['BUCKET_NAME']
-        endpoint = 'oss-' + context.region + '-internal.aliyuncs.com'
+        backup_origin = os.environ['BACKUP_ORIGIN']
+        m = re.search(r'(.*).oss-(.*).aliyuncs.com', backup_origin)
+        if m is None:
+            LOGGER.error(
+                "Backup origin {} is not a valid oss domain".format(backup_origin))
+            raise ValueError(
+                "Backup origin {} is not a valid oss domain".format(backup_origin))
+
+        bucket_name = m.group(1)
+        region = m.group(2)
+        endpoint = 'oss-' + region + '.aliyuncs.com:'
         # endpoint = 'oss-cn-hangzhou-internal.aliyuncs.com'
         bucket = oss2.Bucket(oss_auth, endpoint, bucket_name)
         # backup
         for tmp_file in downloaded_list:
             # /tmp/www.cgbchina.com.cn/index.html -> www.cgbchina.com.cn/index.html
             object_name = tmp_file[5:]
-            LOGGER.info("start to backup matched file = {}".format(object_name))
-            bucket.put_object_from_file(object_name, tmp_file)
+            LOGGER.info(
+                "start to backup matched file = {}".format(object_name))
+            if bucket is not None:
+                bucket.put_object_from_file(object_name, tmp_file)
 
             # 预热目标到CDN域名
-            if "WARMUP_DOMAIN" in os.environ and os.environ["WARMUP_DOMAIN"] != 'dummy': 
+            if "WARMUP_DOMAIN" in os.environ:
                 cdn_domain = os.environ["WARMUP_DOMAIN"]
                 cdn_auth = StsTokenCredential(
                     creds.access_key_id,
                     creds.access_key_secret,
                     creds.security_token)
-                client = AcsClient(region_id=context.region, credential=cdn_auth)
-            
+                client = AcsClient(region_id=context.region,
+                                   credential=cdn_auth)
+
                 request = CommonRequest()
                 request.set_accept_format('json')
                 request.set_domain('cdn.aliyuncs.com')
                 request.set_method('POST')
-                request.set_protocol_type('https') # https | http
+                request.set_protocol_type('https')  # https | http
                 request.set_version('2018-05-10')
                 request.set_action_name('PushObjectCache')
-                cdn_url = cdn_domain + urllib.parse.urlparse('http://' + object_name).path
+                cdn_url = cdn_domain + \
+                    urllib.parse.urlparse('http://' + object_name).path
                 request.add_query_param('ObjectPath', cdn_url)
-                
+
                 response = client.do_action(request)
-                LOGGER.info('PushObjectCache: ' + cdn_url + ', and returned: ' + str(response, encoding = 'utf-8'))
+                LOGGER.info('PushObjectCache: ' + cdn_url +
+                            ', and returned: ' + str(response, encoding='utf-8'))
 
-    return 'success'
+    return dict(download_num=len(downloaded_list), success=True)

src/s.yaml

@@ -1,19 +1,18 @@
 edition: 1.0.0
-name: cdn-backup-source
+name: cdn-backup-origin
 access: '{{ access }}'
 
 vars:
   region: '{{ region }}'
   environmentVariables: 
-    SOURCE_URL: '{{ sourceUrl }}'
-    BUCKET_NAME: '{{ bucketName }}'
-    #WARMUP_DOMAIN: '{{ warmupDomain }}'
+    ORIGIN: '{{ origin }}'
+    BACK_ORIGIN: '{{ backupOrigin }}'
+    WARMUP_DOMAIN: '{{ warmupDomain }}'
   service:
     name: '{{ serviceName }}'
     internetAccess: true
     role: '{{ roleArn }}'
     logConfig: auto
-  cron: '{{ cron }}'
 
 services:
   cron_cdn_backup:
@@ -22,24 +21,20 @@ services:
       region: ${vars.region}
       service: ${vars.service}
       function:
+        name: '{{ functionName }}'
         environmentVariables: ${vars.environmentVariables}
         handler: index.handler
         instanceConcurrency: 1
         instanceType: e1
         memorySize: 512
         runtime: python3
-        timeout: 6000
-        name: '{{ functionName }}'
-        asyncConfiguration:
-          maxAsyncEventAgeInSeconds: 172800
-          maxAsyncRetryAttempts: 1
-          statefulInvocation: false
+        timeout: 1200
         codeUri: ./code
       triggers:
         - name: cron
           type: timer
           qualifier: LATEST
           config:
             payload: ''
-            cronExpression: 'CRON_TZ=Asia/Shanghai ${vars.cron}'
-            enable: true
+            cronExpression: '{{ cronExpression }}'
+            enable: '{{ enable }}'