diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 168e89b..e4b1e31 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,7 +5,9 @@ stages: image: alpine:latest stage: deploy script: - - chmod +x ./package.sh && ./package.sh + - chmod +x ./.devcontainer/initialize.sh + - ./.devcontainer/initialize.sh + - go run ./release.go artifacts: expire_in: 1 week @@ -18,4 +20,4 @@ timer: rules: - if: $CI_PIPELINE_SOURCE == "schedule" when: manual - <<: *build + <<: *build \ No newline at end of file diff --git a/package.sh b/package.sh deleted file mode 100755 index 421bafa..0000000 --- a/package.sh +++ /dev/null @@ -1,131 +0,0 @@ -apk add curl git git-lfs icu-data-full jq npm uuidgen sed >/dev/null -git config --global http.sslVerify false -git config --global advice.detachedHead false - -Repo_Process() { - # Prepare Variables - local UUID=$(uuidgen) - local WORK_ROOT=$1 - local REPO_SETTING=$2 - local ADDR=$(echo $REPO_SETTING | jq -r '.addr') - local DIR=$(echo $REPO_SETTING | jq -r '.dir') - local TAG_REGEX=$(echo $REPO_SETTING | jq -r '.tagRegex') - local IGNORE_REPO=$(echo $REPO_SETTING | jq -r '.ignore') - - local REPO_NAME=$(basename $ADDR | sed 's/\.[^\.]*$//') - local AUTHOR_NAME=$(echo "$ADDR" | awk -F[/:] '{print $(NF-1)}') - local PACKAGE_NAME=$(basename $DIR) - - local PACKAGE_ROOT="$WORK_ROOT/$REPO_NAME-$UUID" - local PACKAGE_DIR="$WORK_ROOT/$REPO_NAME-$UUID/$DIR" - - # Check if ignore - if [[ "$IGNORE_REPO" == "true" ]]; then - return - fi - - if [[ "$TAG_REGEX" == null ]]; then - local TAG_REGEX="^[0-9]+\.[0-9]+\.[0-9]+$" - fi - - if [[ "$DIR" == null ]]; then - local DIR="." - fi - - # Download latest version - cd $WORK_ROOT - git clone $ADDR $REPO_NAME-$UUID >/dev/null 2>&1 - cd $PACKAGE_ROOT - - local TAGS=$(git tag -l --sort authordate) - - # Tag Process - for TAG in $TAGS; do - - # Check if tag is valid - if [[ ! $TAG =~ $TAG_REGEX ]]; then - echo "$REPO_NAME:$DIR@$TAG is invalid tag." - continue - fi - - # Prepare Variables - local GITHUB_PACKAGE_ADDR="https://raw.githubusercontent.com/${AUTHOR_NAME}/${REPO_NAME}/${TAG}/${DIR}/package.json" - - # 404 Check - local GITHUB_STAT=$(curl -sL $GITHUB_PACKAGE_ADDR -o /dev/null -w '%{http_code}\n') - if [[ "$GITHUB_STAT" != "200" ]]; then - echo "$REPO_NAME:$DIR@$TAG has invalid path." - continue - fi - - local PACKAGE_JSON=$(curl -sL $GITHUB_PACKAGE_ADDR) - local GITHUB_NAME=$(echo "$PACKAGE_JSON" | jq -r .name) - local SEM_VERSION=$(echo "$TAG" | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+') - - # Get Server tag - local SVR_STAT=$(curl -sL https://$NPM_REGISTRY/$GITHUB_NAME -o /dev/null -w '%{http_code}\n') - local SVR_VERS="" - if [[ "$SVR_STAT" == "200" ]]; then - local SVR_DATA=$(curl -sL https://$NPM_REGISTRY/$GITHUB_NAME) - local SVR_VERS=$(echo $SVR_DATA | jq .versions | jq -s -r '[ .[] | keys ] | flatten | unique | .[]') - fi - - # Check server version - local EXIST_ON_SVR="false" - for SVR_VER in $SVR_VERS; do - if [[ "$SVR_VER" == "$SEM_VERSION" ]]; then - local EXIST_ON_SVR="true" - break - fi - done - if [[ "$EXIST_ON_SVR" == "true" ]]; then - echo "$REPO_NAME:$DIR@$TAG is already published." - continue - fi - - # Download Repository - cd $WORK_ROOT - rm -r $PACKAGE_ROOT - git clone -b $TAG $ADDR $REPO_NAME-$UUID >/dev/null 2>&1 - cd $PACKAGE_ROOT - git submodule update --init --recursive >/dev/null 2>&1 - git lfs pull >/dev/null 2>&1 - - cd $PACKAGE_DIR - - # Change Version - local PROCESS_VER='.version="'$SEM_VERSION'"' - local PACKAGE_JSON=$(echo "$PACKAGE_JSON" | jq "$PROCESS_VER" | jq -c .) - - # Update package.json - rm package.json - echo $PACKAGE_JSON >package.json - - # Publish Package - local DATA='{"name":"'"$NPM_USER"'", "password":"'"$NPM_PASS"'"}' - local USER="$NPM_USER":"$NPM_PASS" - local RETURN=$(curl -sL -H "Accept:application/json" -H "Content-Type:application/json" -X PUT --data "$DATA" --user "$USER" https://$NPM_REGISTRY/-/user/org.couchdb.user:$NPM_USER) - local TOKEN=$(echo $RETURN | jq '.token') - npm set //$NPM_REGISTRY/:_authToken $TOKEN - npm publish --registry https://$NPM_REGISTRY/ >/dev/null 2>&1 - - if [[ $? -ne 0 ]]; then - echo "$REPO_NAME:$DIR@$TAG failed to publish." - else - echo "$REPO_NAME:$DIR@$TAG published." - fi - - done - - cd $WORK_ROOT - rm -r $PACKAGE_ROOT -} - -JSON_FILES=$(find ./repos -name "*.json") -for JSON_FILE in $JSON_FILES; do - ROOT=$(pwd) - OBJ=$(cat $JSON_FILE | jq -c .) - Repo_Process "$ROOT" "$OBJ" & -done - -wait diff --git a/release.go b/release.go new file mode 100644 index 0000000..b8a9fe3 --- /dev/null +++ b/release.go @@ -0,0 +1,402 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + + "github.com/google/uuid" + "github.com/joho/godotenv" +) + +type RepoType int + +const ( + Unknown RepoType = iota + GitHub + GitLab + BitBucket +) + +type RepoSetting struct { + Addr string `json:"addr"` + Dir string `json:"dir"` + TagRegex string `json:"tagRegex"` +} + +// RepoType と ドメインのマッピング +var repoTypeMap = map[RepoType]string{ + GitHub: "https://raw.githubusercontent.com", +} + +func main() { + godotenv.Load(".env") + npmUser := os.Getenv("NPM_USER") + npmPass := os.Getenv("NPM_PASS") + npmRegistry := os.Getenv("NPM_REGISTRY") + isDevMode := os.Getenv("DEV_MODE") == "true" + + jsonFiles, _ := filepath.Glob("./repos/*.json") + + token, err := getNpmAuthToken(npmRegistry, npmUser, npmPass) + if err != nil { + fmt.Printf("[Error] NPM Auth Failed. %s\n", err) + return + } + + var wg sync.WaitGroup + for _, jsonFile := range jsonFiles { + root, err := os.Getwd() + if err != nil { + fmt.Printf("[Error] Root Directory Get Failed. %s\n", err) + continue + } + fileContent, err := os.ReadFile(jsonFile) + if err != nil { + fmt.Printf("[Error] File Read Failed. %s\n", err) + continue + } + var repoSetting RepoSetting + if err := json.Unmarshal(fileContent, &repoSetting); err != nil { + fmt.Printf("[Error] JSON Parse Failed. %s\n", err) + continue + } + wg.Add(1) + go processRepository(root, repoSetting, token, npmRegistry, &wg, isDevMode) + } + wg.Wait() +} + +func processRepository(workRoot string, repoSetting RepoSetting, npmToken, npmRegistry string, wg *sync.WaitGroup, devMode bool) { + defer wg.Done() + + // 変数の準備 + uuid := uuid.New().String() + addr := repoSetting.Addr + dir := repoSetting.Dir + tagRegex := repoSetting.TagRegex + + repositoryName := strings.TrimSuffix(filepath.Base(addr), filepath.Ext(addr)) + repositoryAuthor := strings.Split(addr, "/")[len(strings.Split(addr, "/"))-2] + + repositoryDirName := fmt.Sprintf("_target_%s-%s", repositoryName, uuid) + repositoryRoot := filepath.Join(workRoot, repositoryDirName) + packageRoot := filepath.Join(repositoryRoot, dir) + + var npmVersions map[string][]string = make(map[string][]string) + + var repositoryType RepoType = Unknown + if strings.HasPrefix(addr, "https://github.com/") { + repositoryType = GitHub + } + + if tagRegex == "" { + tagRegex = "^[0-9]+\\.[0-9]+\\.[0-9]+$" + } + + if dir == "" { + dir = "." + } + + // Tagの確認 + remoteTags, err := getRemoteTags(addr) + if err != nil { + fmt.Printf("[Error] Invalid Repository. %s\n", addr) + return + } + + var validTags []string + + // 有効なTagのみを抽出 + for _, tag := range remoteTags { + matched, _ := regexp.MatchString(tagRegex, tag) + if !matched { + fmt.Printf("[Warn] Invalid Tag. %s:%s@%s\n", repositoryName, dir, tag) + } else { + validTags = append(validTags, tag) + } + } + + var unpublishedTags []string + tagChan := make(chan string) + + // 非同期に未アップロードのタグを抽出 + for _, tag := range validTags { + go func(tag string) { + repositoryJSON, err := getPackageJson(repositoryType, repositoryAuthor, repositoryName, tag, dir) + if err != nil { + fmt.Printf("[Warn] Invalid Path. %s:%s@%s\n", repositoryName, dir, tag) + tagChan <- "" + return + } + + packageName := repositoryJSON["name"].(string) + packageVersion := regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`).FindString(tag) + + if npmVersions[packageName] == nil { + npmVersions[packageName], err = getNpmTags("https://"+npmRegistry, packageName) + if err != nil { + fmt.Printf("[Warn] Invalid Repository. %s\n", err) + // 未アップロードなだけなのでエラーとしない + } + } + + // サーバーに存在するか確認 + existOnServer := false + for _, npmVersion := range npmVersions[packageName] { + if npmVersion == packageVersion { + existOnServer = true + break + } + } + + if existOnServer { + fmt.Printf("[Info] Already Published. %s:%s@%s\n", repositoryName, dir, tag) + tagChan <- "" + } else { + tagChan <- tag + } + }(tag) + } + + // チャネルから結果を受け取る + for range validTags { + tag := <-tagChan + if tag != "" { + unpublishedTags = append(unpublishedTags, tag) + } + } + + unpublishedTags = sortTags(unpublishedTags) + + // 未アップロードのパッケージを処理 + for _, tag := range unpublishedTags { + + if err := os.Chdir(workRoot); err != nil { + fmt.Printf("[Error] Work Directory Change Failed. %s\n", err) + continue + } + + // repositoryDirName ディレクトリが無ければtagタグをClone, あればtagタグをチェックアウト + if _, err := os.Stat(repositoryRoot); os.IsNotExist(err) { + if err := exec.Command("git", "clone", "-b", tag, addr, repositoryDirName).Run(); err != nil { + fmt.Printf("[Error] clone failed. %s\n", err) + continue + } + if err := os.Chdir(repositoryRoot); err != nil { + fmt.Printf("[Error] directory change failed. %s\n", err) + continue + } + } else { + if err := os.Chdir(repositoryRoot); err != nil { + fmt.Printf("[Error] directory change failed. %s\n", err) + continue + } + if err := exec.Command("git", "checkout", tag).Run(); err != nil { + fmt.Printf("[Error] checkout failed. %s\n", err) + continue + } + } + + if err := exec.Command("git", "submodule", "update", "--init", "--recursive").Run(); err != nil { + fmt.Printf("[Error] submodule update failed. %s\n", err) + continue + } + if err := exec.Command("git", "lfs", "pull").Run(); err != nil { + fmt.Printf("[Error] lfs pull failed. %s\n", err) + continue + } + + if err := os.Chdir(packageRoot); err != nil { + fmt.Printf("[Error] directory change failed. %s\n", err) + continue + } + + // パッケージのアップロード + if err := exec.Command("npm", "set", fmt.Sprintf("//%s/:_authToken", npmRegistry), npmToken).Run(); err != nil { + fmt.Printf("[Error] npm auth failed. %s\n", err) + continue + } + if !devMode { + if err := exec.Command("npm", "publish", "--registry", fmt.Sprintf("https://%s/", npmRegistry)).Run(); err != nil { + fmt.Printf("%s:%s@%s failed to publish. %s\n", repositoryName, dir, tag, err) + } else { + fmt.Printf("%s:%s@%s published.\n", repositoryName, dir, tag) + } + } else { + fmt.Printf("%s:%s@%s dummy published.\n", repositoryName, dir, tag) + } + } + + // 作業ディレクトリの削除 + if err := os.Chdir(workRoot); err != nil { + fmt.Printf("[Error] Work Directory Change Failed. %s\n", err) + } + if err := os.RemoveAll(repositoryRoot); err != nil { + fmt.Printf("[Error] Work Directory Remove Failed. %s\n", err) + } +} + +// タグをソートする関数です。 +func sortTags(tags []string) []string { + sort.Slice(tags, func(i, j int) bool { + versionI := strings.Split(tags[i], ".") + versionJ := strings.Split(tags[j], ".") + + for k := 0; k < len(versionI) && k < len(versionJ); k++ { + numI, _ := strconv.Atoi(versionI[k]) + numJ, _ := strconv.Atoi(versionJ[k]) + if numI != numJ { + return numI < numJ + } + } + return len(versionI) < len(versionJ) + }) + + return tags +} + +// 指定されたリポジトリのタグ一覧を取得する関数です。 +func getRemoteTags(repoAddr string) ([]string, error) { + // git ls-remote --tags コマンドを実行 + cmd := exec.Command("git", "ls-remote", "--tags", repoAddr) + out, err := cmd.Output() + if err != nil { + return nil, err + } + + // 出力をパースしてタグを抽出 + lines := strings.Split(string(out), "\n") + var tags []string + for _, line := range lines { + if line == "" { + continue + } + parts := strings.Split(line, "\t") + if len(parts) < 2 { + continue + } + tag := strings.Replace(parts[1], "refs/tags/", "", 1) + tags = append(tags, tag) + } + + return tags, nil +} + +// 指定された npm レジストリとパッケージ名を使用して、npm パッケージのタグ一覧を取得する関数です。 +func getNpmTags(npmRepo string, packageName string) ([]string, error) { + var url = fmt.Sprintf("%s/%s", npmRepo, packageName) + + // HTTP GET リクエストを送信 + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // レスポンスのボディを読み取る + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + // JSONをパースしてmapに変換 + var result map[string]interface{} + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + + if result["versions"] == nil { + return nil, fmt.Errorf("versions not found") + } + + // バージョン情報一覧を取得 + var versions []string + for version := range result["versions"].(map[string]interface{}) { + versions = append(versions, version) + } + + return versions, nil +} + +// 指定されたリポジトリの package.json を取得する関数です。 +func getPackageJson(repoType RepoType, author string, repository string, tag string, dir string) (map[string]interface{}, error) { + var url = fmt.Sprintf("%s/%s/%s/%s/%s/package.json", repoTypeMap[repoType], author, repository, tag, dir) + + // HTTP GET リクエストを送信 + resp, err := http.Get(url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + // レスポンスのボディを読み取る + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + // JSONをパースしてmapに変換 + var result map[string]interface{} + if err := json.Unmarshal(body, &result); err != nil { + return nil, err + } + + return result, nil +} + +// 指定された npm レジストリ、ユーザー名、およびパスワードを使用してnpm 認証トークンを取得する関数です。 +func getNpmAuthToken(npmRegistry, npmUser, npmPass string) (string, error) { + data := map[string]string{ + "name": npmUser, + "password": npmPass, + } + jsonData, err := json.Marshal(data) + if err != nil { + return "", err + } + + req, err := http.NewRequest("PUT", fmt.Sprintf("https://%s/-/user/org.couchdb.user:%s", npmRegistry, npmUser), bytes.NewBuffer(jsonData)) + if err != nil { + return "", err + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth(npmUser, npmPass) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + return "", fmt.Errorf("failed to get token, status code: %d", resp.StatusCode) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + var returnData map[string]interface{} + if err := json.Unmarshal(body, &returnData); err != nil { + return "", err + } + token, ok := returnData["token"].(string) + if !ok { + return "", fmt.Errorf("token not found in response") + } + return token, nil +}