処理をGoへ移植
This commit is contained in:
@@ -5,7 +5,9 @@ stages:
|
||||
image: alpine:latest
|
||||
stage: deploy
|
||||
script:
|
||||
- chmod +x ./package.sh && ./package.sh
|
||||
- chmod +x ./.devcontainer/initialize.sh
|
||||
- ./.devcontainer/initialize.sh
|
||||
- go run ./release.go
|
||||
artifacts:
|
||||
expire_in: 1 week
|
||||
|
||||
@@ -18,4 +20,4 @@ timer:
|
||||
rules:
|
||||
- if: $CI_PIPELINE_SOURCE == "schedule"
|
||||
when: manual
|
||||
<<: *build
|
||||
<<: *build
|
||||
131
package.sh
131
package.sh
@@ -1,131 +0,0 @@
|
||||
apk add curl git git-lfs icu-data-full jq npm uuidgen sed >/dev/null
|
||||
git config --global http.sslVerify false
|
||||
git config --global advice.detachedHead false
|
||||
|
||||
Repo_Process() {
|
||||
# Prepare Variables
|
||||
local UUID=$(uuidgen)
|
||||
local WORK_ROOT=$1
|
||||
local REPO_SETTING=$2
|
||||
local ADDR=$(echo $REPO_SETTING | jq -r '.addr')
|
||||
local DIR=$(echo $REPO_SETTING | jq -r '.dir')
|
||||
local TAG_REGEX=$(echo $REPO_SETTING | jq -r '.tagRegex')
|
||||
local IGNORE_REPO=$(echo $REPO_SETTING | jq -r '.ignore')
|
||||
|
||||
local REPO_NAME=$(basename $ADDR | sed 's/\.[^\.]*$//')
|
||||
local AUTHOR_NAME=$(echo "$ADDR" | awk -F[/:] '{print $(NF-1)}')
|
||||
local PACKAGE_NAME=$(basename $DIR)
|
||||
|
||||
local PACKAGE_ROOT="$WORK_ROOT/$REPO_NAME-$UUID"
|
||||
local PACKAGE_DIR="$WORK_ROOT/$REPO_NAME-$UUID/$DIR"
|
||||
|
||||
# Check if ignore
|
||||
if [[ "$IGNORE_REPO" == "true" ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ "$TAG_REGEX" == null ]]; then
|
||||
local TAG_REGEX="^[0-9]+\.[0-9]+\.[0-9]+$"
|
||||
fi
|
||||
|
||||
if [[ "$DIR" == null ]]; then
|
||||
local DIR="."
|
||||
fi
|
||||
|
||||
# Download latest version
|
||||
cd $WORK_ROOT
|
||||
git clone $ADDR $REPO_NAME-$UUID >/dev/null 2>&1
|
||||
cd $PACKAGE_ROOT
|
||||
|
||||
local TAGS=$(git tag -l --sort authordate)
|
||||
|
||||
# Tag Process
|
||||
for TAG in $TAGS; do
|
||||
|
||||
# Check if tag is valid
|
||||
if [[ ! $TAG =~ $TAG_REGEX ]]; then
|
||||
echo "$REPO_NAME:$DIR@$TAG is invalid tag."
|
||||
continue
|
||||
fi
|
||||
|
||||
# Prepare Variables
|
||||
local GITHUB_PACKAGE_ADDR="https://raw.githubusercontent.com/${AUTHOR_NAME}/${REPO_NAME}/${TAG}/${DIR}/package.json"
|
||||
|
||||
# 404 Check
|
||||
local GITHUB_STAT=$(curl -sL $GITHUB_PACKAGE_ADDR -o /dev/null -w '%{http_code}\n')
|
||||
if [[ "$GITHUB_STAT" != "200" ]]; then
|
||||
echo "$REPO_NAME:$DIR@$TAG has invalid path."
|
||||
continue
|
||||
fi
|
||||
|
||||
local PACKAGE_JSON=$(curl -sL $GITHUB_PACKAGE_ADDR)
|
||||
local GITHUB_NAME=$(echo "$PACKAGE_JSON" | jq -r .name)
|
||||
local SEM_VERSION=$(echo "$TAG" | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+')
|
||||
|
||||
# Get Server tag
|
||||
local SVR_STAT=$(curl -sL https://$NPM_REGISTRY/$GITHUB_NAME -o /dev/null -w '%{http_code}\n')
|
||||
local SVR_VERS=""
|
||||
if [[ "$SVR_STAT" == "200" ]]; then
|
||||
local SVR_DATA=$(curl -sL https://$NPM_REGISTRY/$GITHUB_NAME)
|
||||
local SVR_VERS=$(echo $SVR_DATA | jq .versions | jq -s -r '[ .[] | keys ] | flatten | unique | .[]')
|
||||
fi
|
||||
|
||||
# Check server version
|
||||
local EXIST_ON_SVR="false"
|
||||
for SVR_VER in $SVR_VERS; do
|
||||
if [[ "$SVR_VER" == "$SEM_VERSION" ]]; then
|
||||
local EXIST_ON_SVR="true"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [[ "$EXIST_ON_SVR" == "true" ]]; then
|
||||
echo "$REPO_NAME:$DIR@$TAG is already published."
|
||||
continue
|
||||
fi
|
||||
|
||||
# Download Repository
|
||||
cd $WORK_ROOT
|
||||
rm -r $PACKAGE_ROOT
|
||||
git clone -b $TAG $ADDR $REPO_NAME-$UUID >/dev/null 2>&1
|
||||
cd $PACKAGE_ROOT
|
||||
git submodule update --init --recursive >/dev/null 2>&1
|
||||
git lfs pull >/dev/null 2>&1
|
||||
|
||||
cd $PACKAGE_DIR
|
||||
|
||||
# Change Version
|
||||
local PROCESS_VER='.version="'$SEM_VERSION'"'
|
||||
local PACKAGE_JSON=$(echo "$PACKAGE_JSON" | jq "$PROCESS_VER" | jq -c .)
|
||||
|
||||
# Update package.json
|
||||
rm package.json
|
||||
echo $PACKAGE_JSON >package.json
|
||||
|
||||
# Publish Package
|
||||
local DATA='{"name":"'"$NPM_USER"'", "password":"'"$NPM_PASS"'"}'
|
||||
local USER="$NPM_USER":"$NPM_PASS"
|
||||
local RETURN=$(curl -sL -H "Accept:application/json" -H "Content-Type:application/json" -X PUT --data "$DATA" --user "$USER" https://$NPM_REGISTRY/-/user/org.couchdb.user:$NPM_USER)
|
||||
local TOKEN=$(echo $RETURN | jq '.token')
|
||||
npm set //$NPM_REGISTRY/:_authToken $TOKEN
|
||||
npm publish --registry https://$NPM_REGISTRY/ >/dev/null 2>&1
|
||||
|
||||
if [[ $? -ne 0 ]]; then
|
||||
echo "$REPO_NAME:$DIR@$TAG failed to publish."
|
||||
else
|
||||
echo "$REPO_NAME:$DIR@$TAG published."
|
||||
fi
|
||||
|
||||
done
|
||||
|
||||
cd $WORK_ROOT
|
||||
rm -r $PACKAGE_ROOT
|
||||
}
|
||||
|
||||
JSON_FILES=$(find ./repos -name "*.json")
|
||||
for JSON_FILE in $JSON_FILES; do
|
||||
ROOT=$(pwd)
|
||||
OBJ=$(cat $JSON_FILE | jq -c .)
|
||||
Repo_Process "$ROOT" "$OBJ" &
|
||||
done
|
||||
|
||||
wait
|
||||
402
release.go
Normal file
402
release.go
Normal file
@@ -0,0 +1,402 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/joho/godotenv"
|
||||
)
|
||||
|
||||
type RepoType int
|
||||
|
||||
const (
|
||||
Unknown RepoType = iota
|
||||
GitHub
|
||||
GitLab
|
||||
BitBucket
|
||||
)
|
||||
|
||||
type RepoSetting struct {
|
||||
Addr string `json:"addr"`
|
||||
Dir string `json:"dir"`
|
||||
TagRegex string `json:"tagRegex"`
|
||||
}
|
||||
|
||||
// RepoType と ドメインのマッピング
|
||||
var repoTypeMap = map[RepoType]string{
|
||||
GitHub: "https://raw.githubusercontent.com",
|
||||
}
|
||||
|
||||
func main() {
|
||||
godotenv.Load(".env")
|
||||
npmUser := os.Getenv("NPM_USER")
|
||||
npmPass := os.Getenv("NPM_PASS")
|
||||
npmRegistry := os.Getenv("NPM_REGISTRY")
|
||||
isDevMode := os.Getenv("DEV_MODE") == "true"
|
||||
|
||||
jsonFiles, _ := filepath.Glob("./repos/*.json")
|
||||
|
||||
token, err := getNpmAuthToken(npmRegistry, npmUser, npmPass)
|
||||
if err != nil {
|
||||
fmt.Printf("[Error] NPM Auth Failed. %s\n", err)
|
||||
return
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, jsonFile := range jsonFiles {
|
||||
root, err := os.Getwd()
|
||||
if err != nil {
|
||||
fmt.Printf("[Error] Root Directory Get Failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
fileContent, err := os.ReadFile(jsonFile)
|
||||
if err != nil {
|
||||
fmt.Printf("[Error] File Read Failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
var repoSetting RepoSetting
|
||||
if err := json.Unmarshal(fileContent, &repoSetting); err != nil {
|
||||
fmt.Printf("[Error] JSON Parse Failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go processRepository(root, repoSetting, token, npmRegistry, &wg, isDevMode)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func processRepository(workRoot string, repoSetting RepoSetting, npmToken, npmRegistry string, wg *sync.WaitGroup, devMode bool) {
|
||||
defer wg.Done()
|
||||
|
||||
// 変数の準備
|
||||
uuid := uuid.New().String()
|
||||
addr := repoSetting.Addr
|
||||
dir := repoSetting.Dir
|
||||
tagRegex := repoSetting.TagRegex
|
||||
|
||||
repositoryName := strings.TrimSuffix(filepath.Base(addr), filepath.Ext(addr))
|
||||
repositoryAuthor := strings.Split(addr, "/")[len(strings.Split(addr, "/"))-2]
|
||||
|
||||
repositoryDirName := fmt.Sprintf("_target_%s-%s", repositoryName, uuid)
|
||||
repositoryRoot := filepath.Join(workRoot, repositoryDirName)
|
||||
packageRoot := filepath.Join(repositoryRoot, dir)
|
||||
|
||||
var npmVersions map[string][]string = make(map[string][]string)
|
||||
|
||||
var repositoryType RepoType = Unknown
|
||||
if strings.HasPrefix(addr, "https://github.com/") {
|
||||
repositoryType = GitHub
|
||||
}
|
||||
|
||||
if tagRegex == "" {
|
||||
tagRegex = "^[0-9]+\\.[0-9]+\\.[0-9]+$"
|
||||
}
|
||||
|
||||
if dir == "" {
|
||||
dir = "."
|
||||
}
|
||||
|
||||
// Tagの確認
|
||||
remoteTags, err := getRemoteTags(addr)
|
||||
if err != nil {
|
||||
fmt.Printf("[Error] Invalid Repository. %s\n", addr)
|
||||
return
|
||||
}
|
||||
|
||||
var validTags []string
|
||||
|
||||
// 有効なTagのみを抽出
|
||||
for _, tag := range remoteTags {
|
||||
matched, _ := regexp.MatchString(tagRegex, tag)
|
||||
if !matched {
|
||||
fmt.Printf("[Warn] Invalid Tag. %s:%s@%s\n", repositoryName, dir, tag)
|
||||
} else {
|
||||
validTags = append(validTags, tag)
|
||||
}
|
||||
}
|
||||
|
||||
var unpublishedTags []string
|
||||
tagChan := make(chan string)
|
||||
|
||||
// 非同期に未アップロードのタグを抽出
|
||||
for _, tag := range validTags {
|
||||
go func(tag string) {
|
||||
repositoryJSON, err := getPackageJson(repositoryType, repositoryAuthor, repositoryName, tag, dir)
|
||||
if err != nil {
|
||||
fmt.Printf("[Warn] Invalid Path. %s:%s@%s\n", repositoryName, dir, tag)
|
||||
tagChan <- ""
|
||||
return
|
||||
}
|
||||
|
||||
packageName := repositoryJSON["name"].(string)
|
||||
packageVersion := regexp.MustCompile(`[0-9]+\.[0-9]+\.[0-9]+`).FindString(tag)
|
||||
|
||||
if npmVersions[packageName] == nil {
|
||||
npmVersions[packageName], err = getNpmTags("https://"+npmRegistry, packageName)
|
||||
if err != nil {
|
||||
fmt.Printf("[Warn] Invalid Repository. %s\n", err)
|
||||
// 未アップロードなだけなのでエラーとしない
|
||||
}
|
||||
}
|
||||
|
||||
// サーバーに存在するか確認
|
||||
existOnServer := false
|
||||
for _, npmVersion := range npmVersions[packageName] {
|
||||
if npmVersion == packageVersion {
|
||||
existOnServer = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if existOnServer {
|
||||
fmt.Printf("[Info] Already Published. %s:%s@%s\n", repositoryName, dir, tag)
|
||||
tagChan <- ""
|
||||
} else {
|
||||
tagChan <- tag
|
||||
}
|
||||
}(tag)
|
||||
}
|
||||
|
||||
// チャネルから結果を受け取る
|
||||
for range validTags {
|
||||
tag := <-tagChan
|
||||
if tag != "" {
|
||||
unpublishedTags = append(unpublishedTags, tag)
|
||||
}
|
||||
}
|
||||
|
||||
unpublishedTags = sortTags(unpublishedTags)
|
||||
|
||||
// 未アップロードのパッケージを処理
|
||||
for _, tag := range unpublishedTags {
|
||||
|
||||
if err := os.Chdir(workRoot); err != nil {
|
||||
fmt.Printf("[Error] Work Directory Change Failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// repositoryDirName ディレクトリが無ければtagタグをClone, あればtagタグをチェックアウト
|
||||
if _, err := os.Stat(repositoryRoot); os.IsNotExist(err) {
|
||||
if err := exec.Command("git", "clone", "-b", tag, addr, repositoryDirName).Run(); err != nil {
|
||||
fmt.Printf("[Error] clone failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
if err := os.Chdir(repositoryRoot); err != nil {
|
||||
fmt.Printf("[Error] directory change failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
if err := os.Chdir(repositoryRoot); err != nil {
|
||||
fmt.Printf("[Error] directory change failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
if err := exec.Command("git", "checkout", tag).Run(); err != nil {
|
||||
fmt.Printf("[Error] checkout failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err := exec.Command("git", "submodule", "update", "--init", "--recursive").Run(); err != nil {
|
||||
fmt.Printf("[Error] submodule update failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
if err := exec.Command("git", "lfs", "pull").Run(); err != nil {
|
||||
fmt.Printf("[Error] lfs pull failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err := os.Chdir(packageRoot); err != nil {
|
||||
fmt.Printf("[Error] directory change failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// パッケージのアップロード
|
||||
if err := exec.Command("npm", "set", fmt.Sprintf("//%s/:_authToken", npmRegistry), npmToken).Run(); err != nil {
|
||||
fmt.Printf("[Error] npm auth failed. %s\n", err)
|
||||
continue
|
||||
}
|
||||
if !devMode {
|
||||
if err := exec.Command("npm", "publish", "--registry", fmt.Sprintf("https://%s/", npmRegistry)).Run(); err != nil {
|
||||
fmt.Printf("%s:%s@%s failed to publish. %s\n", repositoryName, dir, tag, err)
|
||||
} else {
|
||||
fmt.Printf("%s:%s@%s published.\n", repositoryName, dir, tag)
|
||||
}
|
||||
} else {
|
||||
fmt.Printf("%s:%s@%s dummy published.\n", repositoryName, dir, tag)
|
||||
}
|
||||
}
|
||||
|
||||
// 作業ディレクトリの削除
|
||||
if err := os.Chdir(workRoot); err != nil {
|
||||
fmt.Printf("[Error] Work Directory Change Failed. %s\n", err)
|
||||
}
|
||||
if err := os.RemoveAll(repositoryRoot); err != nil {
|
||||
fmt.Printf("[Error] Work Directory Remove Failed. %s\n", err)
|
||||
}
|
||||
}
|
||||
|
||||
// タグをソートする関数です。
|
||||
func sortTags(tags []string) []string {
|
||||
sort.Slice(tags, func(i, j int) bool {
|
||||
versionI := strings.Split(tags[i], ".")
|
||||
versionJ := strings.Split(tags[j], ".")
|
||||
|
||||
for k := 0; k < len(versionI) && k < len(versionJ); k++ {
|
||||
numI, _ := strconv.Atoi(versionI[k])
|
||||
numJ, _ := strconv.Atoi(versionJ[k])
|
||||
if numI != numJ {
|
||||
return numI < numJ
|
||||
}
|
||||
}
|
||||
return len(versionI) < len(versionJ)
|
||||
})
|
||||
|
||||
return tags
|
||||
}
|
||||
|
||||
// 指定されたリポジトリのタグ一覧を取得する関数です。
|
||||
func getRemoteTags(repoAddr string) ([]string, error) {
|
||||
// git ls-remote --tags コマンドを実行
|
||||
cmd := exec.Command("git", "ls-remote", "--tags", repoAddr)
|
||||
out, err := cmd.Output()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 出力をパースしてタグを抽出
|
||||
lines := strings.Split(string(out), "\n")
|
||||
var tags []string
|
||||
for _, line := range lines {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(line, "\t")
|
||||
if len(parts) < 2 {
|
||||
continue
|
||||
}
|
||||
tag := strings.Replace(parts[1], "refs/tags/", "", 1)
|
||||
tags = append(tags, tag)
|
||||
}
|
||||
|
||||
return tags, nil
|
||||
}
|
||||
|
||||
// 指定された npm レジストリとパッケージ名を使用して、npm パッケージのタグ一覧を取得する関数です。
|
||||
func getNpmTags(npmRepo string, packageName string) ([]string, error) {
|
||||
var url = fmt.Sprintf("%s/%s", npmRepo, packageName)
|
||||
|
||||
// HTTP GET リクエストを送信
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// レスポンスのボディを読み取る
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// JSONをパースしてmapに変換
|
||||
var result map[string]interface{}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if result["versions"] == nil {
|
||||
return nil, fmt.Errorf("versions not found")
|
||||
}
|
||||
|
||||
// バージョン情報一覧を取得
|
||||
var versions []string
|
||||
for version := range result["versions"].(map[string]interface{}) {
|
||||
versions = append(versions, version)
|
||||
}
|
||||
|
||||
return versions, nil
|
||||
}
|
||||
|
||||
// 指定されたリポジトリの package.json を取得する関数です。
|
||||
func getPackageJson(repoType RepoType, author string, repository string, tag string, dir string) (map[string]interface{}, error) {
|
||||
var url = fmt.Sprintf("%s/%s/%s/%s/%s/package.json", repoTypeMap[repoType], author, repository, tag, dir)
|
||||
|
||||
// HTTP GET リクエストを送信
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
// レスポンスのボディを読み取る
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// JSONをパースしてmapに変換
|
||||
var result map[string]interface{}
|
||||
if err := json.Unmarshal(body, &result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// 指定された npm レジストリ、ユーザー名、およびパスワードを使用してnpm 認証トークンを取得する関数です。
|
||||
func getNpmAuthToken(npmRegistry, npmUser, npmPass string) (string, error) {
|
||||
data := map[string]string{
|
||||
"name": npmUser,
|
||||
"password": npmPass,
|
||||
}
|
||||
jsonData, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("PUT", fmt.Sprintf("https://%s/-/user/org.couchdb.user:%s", npmRegistry, npmUser), bytes.NewBuffer(jsonData))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.SetBasicAuth(npmUser, npmPass)
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusCreated {
|
||||
return "", fmt.Errorf("failed to get token, status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
var returnData map[string]interface{}
|
||||
if err := json.Unmarshal(body, &returnData); err != nil {
|
||||
return "", err
|
||||
}
|
||||
token, ok := returnData["token"].(string)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("token not found in response")
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
Reference in New Issue
Block a user