实现一个简单的Flutter/Dart版本的对象存储（COS）SDK

0x00 简介

本文中，尝试使用dart实现对象存储SDK，目前只实现了listObject、putObject、deleteObject三个功能，足够覆盖简单的增删查场景了。

0x01 核心代码

具体代码可以参考这里

COSConfig

这个类主要是管理一些基础信息，如secretId、secretKey、bucketName、region等：

class COSConfig {
  String secretId;
  String secretKey;
  String bucketName;
  String region;
  String scheme;
  bool anonymous;
  COSConfig(
    this.secretId,
    this.secretKey,
    this.bucketName,
    this.region, {
    this.scheme = "https",
    this.anonymous = false,
  });

  String get uri {
    return "$scheme://$bucketName.cos.$region.myqcloud.com";
  }
}

COSClientBase

这个类主要是负责处理一些通用的逻辑，比如创建请求、给请求签名。

import 'dart:io';

import 'package:convert/convert.dart';
import 'package:crypto/crypto.dart';

import 'cos_comm.dart';
import "cos_config.dart";

class COSClientBase {
  final COSConfig _config;
  COSClientBase(this._config);

  ///生成签名
  String getSign(String method, String key,
      {Map<String, String?> headers = const {},
      Map<String, String?> params = const {},
      DateTime? signTime}) {
    if (_config.anonymous) {
      return "";
    } else {
      signTime = signTime ?? DateTime.now();
      int startSignTime = signTime.millisecondsSinceEpoch ~/ 1000 - 60;
      int stopSignTime = signTime.millisecondsSinceEpoch ~/ 1000 + 120;
      String keyTime = "$startSignTime;$stopSignTime";
      cosLog("keyTime=$keyTime");
      String signKey = hmacSha1(keyTime, _config.secretKey);
      cosLog("signKey=$signKey");

      var lap = getListAndParameters(params);
      String urlParamList = lap[0];
      String httpParameters = lap[1];
      cosLog("urlParamList=$urlParamList");
      cosLog("httpParameters=$httpParameters");

      lap = getListAndParameters(filterHeaders(headers));
      String headerList = lap[0];
      String httpHeaders = lap[1];
      cosLog("headerList=$headerList");
      cosLog("httpHeaders=$httpHeaders");

      String httpString =
          "${method.toLowerCase()}/n$key/n$httpParameters/n$httpHeaders/n";
      cosLog("httpString=$httpString");
      String stringToSign =
          "sha1/n$keyTime/n${hex.encode(sha1.convert(httpString.codeUnits).bytes)}/n";
      cosLog("stringToSign=$stringToSign");
      String signature = hmacSha1(stringToSign, signKey);
      cosLog("signature=$signature");
      String res =
          "q-sign-algorithm=sha1&q-ak=${_config.secretId}&q-sign-time=$keyTime&q-key-time=$keyTime&q-header-list=$headerList&q-url-param-list=$urlParamList&q-signature=$signature";
      cosLog("Authorization=$res");
      return res;
    }
  }

  filterHeaders(Map<String, String?> src) {
    Map<String, String?> res = {};
    res["host"] = src["host"];
    res["accept-encoding"] = src["accept-encoding"];
    return res;
  }

  ///处理请求头和参数列表
  List<String> getListAndParameters(Map<String, String?> params) {
    params = params.map((key, value) => MapEntry(
        Uri.encodeComponent(key).toLowerCase(),
        Uri.encodeComponent(value ?? "")));

    var keys = params.keys.toList();
    keys.sort();
    String urlParamList = keys.join(";");
    String httpParameters =
        keys.map((e) => e + "=" + (params[e] ?? "")).join("&");
    return [urlParamList, httpParameters];
  }

  /// 使用HMAC-SHA1计算摘要
  String hmacSha1(String msg, String key) {
    return hex.encode(Hmac(sha1, key.codeUnits).convert(msg.codeUnits).bytes);
  }

  Future<HttpClientRequest> getRequest(String method, String action,
      {Map<String, String?> params = const {},
      Map<String, String?> headers = const {}}) async {
    String urlParams =
        params.keys.toList().map((e) => e + "=" + (params[e] ?? "")).join("&");
    if (urlParams.isNotEmpty) {
      urlParams = "?" + urlParams;
    }
    HttpClient client = HttpClient();

    if (!action.startsWith("/")) {
      action = "/" + action;
    }

    var req = await client.openUrl(
        method, Uri.parse("${_config.uri}$action$urlParams"));
    headers.forEach((key, value) {
      req.headers.add(key, value ?? "");
    });
    Map<String, String> _headers = {};
    req.headers.forEach((name, values) {
      _headers[name] = values[0];
    });
    var sighn = getSign(method, action, params: params, headers: _headers);
    req.headers.add("Authorization", sighn);
    return req;
  }

  Future<HttpClientResponse> getResponse(String method, String action,
      {Map<String, String?> params = const {},
      Map<String, String?> headers = const {}}) async {
    var req =
        await getRequest(method, action, params: params, headers: headers);
    var res = await req.close();
    return res;
  }
}

COSClient

这个类就实现了listObject、putObject、deleteObject三个功能

import 'dart:convert';
import 'dart:io';

import 'package:xml/xml.dart';

import 'cos_clientbase.dart';
import 'cos_comm.dart';
import "cos_config.dart";
import 'cos_exception.dart';
import "cos_model.dart";

class COSClient extends COSClientBase {
  COSClient(COSConfig _config) : super(_config);

  Future<ListBucketResult> listObject({String prefix = ""}) async {
    cosLog("listObject");
    var response = await getResponse("GET", "/", params: {"prefix": prefix});
    cosLog("request-id:" + (response.headers["x-cos-request-id"]?.first ?? ""));
    String xmlContent = await response.transform(utf8.decoder).join("");
    if (response.statusCode != 200) {
      throw COSException(response.statusCode, xmlContent);
    }
    var content = XmlDocument.parse(xmlContent);
    return ListBucketResult(content.rootElement);
  }

  putObject(String objectKey, String filePath) async {
    cosLog("putObject");
    var f = File(filePath);
    int flength = await f.length();
    var fs = f.openRead();
    var req = await getRequest("PUT", objectKey, headers: {
      "content-type": "image/jpeg",
      "content-length": flength.toString()
    });
    await req.addStream(fs);
    var response = await req.close();
    cosLog("request-id:" + (response.headers["x-cos-request-id"]?.first ?? ""));
    if (response.statusCode != 200) {
      cosLog("putObject error");
      String content = await response.transform(utf8.decoder).join("");
      throw COSException(response.statusCode, content);
    }
  }

  deleteObject(String objectKey) async {
    cosLog("deleteObject");
    var response = await getResponse("DELETE", objectKey);
    cosLog("request-id:" + (response.headers["x-cos-request-id"]?.first ?? ""));
    if (response.statusCode != 204) {
      cosLog("deleteObject error");
      String content = await response.transform(utf8.decoder).join("");
      throw COSException(response.statusCode, content);
    }
  }
}

0x02 例子

  test('COSClient listObject', () async {
    await client.deleteObject("abc/avata.jpg");
    ListBucketResult res = await client.listObject(prefix: "abc/avata.jpg");
    expect(res.contents.length, 0);

    await client.putObject("abc/avata.jpg", "avata.jpg");
    res = await client.listObject(prefix: "abc/avata.jpg");
    expect(res.contents.length, 1);

    await client.deleteObject("avata.jpg");
    res = await client.listObject(prefix: "avata.jpg");
    expect(res.contents.length, 0);
  });

0x03 碰到的坑

开发过程中，遇到的最头疼的问题是调用COS的接口，老是提示我签名错误。

经过排查，初步判断应该是dart本身的HttpClientRequest实现有点问题。

在代码中，遍历HttpClientRequest的header属性得到4个记录：

但是经过我的验证，实际的请求头中，Content-Length变成了空字符串：

让腾讯的小伙伴帮我看了下，他确认没有收到Content-Length这个头，后来我发现，带上User-Agent这个头也会提示签名失败，这点我就不能理解了，没办法，为了保险起见，代码改成只对Host和Accept-Encoding进行签名，接口就通了。

最后经过我测试，好像签名的时候urlParamList和headerList这两个为空也行，但是腾讯的小伙伴说他们有计划改成所有的都需要验证的强验证机制，感觉这样子的话就不太好搞啊。

0x04 另外

另外，我也参考了下Python SDK的源码，官方提供的SDK，在headers验证上面也是做过筛选的。

cos_auth.py中：

def filter_headers(data):
    """只设置host content-type 还有x开头的头部.

    :param data(dict): 所有的头部信息.
    :return(dict): 计算进签名的头部.
    """
    valid_headers = [
        "cache-control",
        "content-disposition",
        "content-encoding",
        "content-type",
        "expires",
        "content-md5",
        "content-length",
        "host"
    ]
    headers = {}
    for i in data:
        if str.lower(i) in valid_headers or str.lower(i[0]) == "x":
            headers[i] = data[i]
    return headers

0x05 最后

其实本来是想实现一个相对完善的SDK的，但是后来发现COS过于强大，功能过于丰富，很多东西我也不太懂，所以就只能先实现个精简版本的了。