diff --git a/.github/workflows/CodeQL.yml b/.github/workflows/CodeQL.yml index 419bc84..b011b5b 100644 --- a/.github/workflows/CodeQL.yml +++ b/.github/workflows/CodeQL.yml @@ -12,14 +12,14 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v4 with: languages: go - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v4 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 \ No newline at end of file + uses: github/codeql-action/analyze@v4 \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6c9c1e7..f8d4a91 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,11 +14,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v6 - name: Docker meta id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: | buddyspencer/gickup @@ -33,29 +33,29 @@ jobs: type=raw,value=dev,enable=${{ github.ref == 'refs/heads/main' }} - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Github Packages if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GHCR_PAT }} - name: Build image and push to Docker Hub and GitHub Container Registry - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . tags: ${{ steps.meta.outputs.tags }} @@ -71,11 +71,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v6 - name: Docker meta id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: | buddyspencer/gickup @@ -89,29 +89,29 @@ jobs: type=raw,value=ubuntu-dev,enable=${{ github.ref == 'refs/heads/main' }} - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Github Packages if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GHCR_PAT }} - name: Build image and push to Docker Hub and GitHub Container Registry - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . file: Dockerfile.ubuntu diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 7c83815..f702525 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -12,10 +12,12 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 + with: + fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v6 with: go-version: 1.26 @@ -23,7 +25,8 @@ jobs: run: go test -v ./... - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 + if: startsWith(github.ref, 'refs/tags/') + uses: goreleaser/goreleaser-action@v7 with: # either 'goreleaser' (default) or 'goreleaser-pro' distribution: goreleaser @@ -31,9 +34,21 @@ jobs: args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Run GoReleaser + if: ${{ !startsWith(github.ref, 'refs/tags/v') }} + uses: goreleaser/goreleaser-action@v7 + with: + # either 'goreleaser' (default) or 'goreleaser-pro' + distribution: goreleaser + version: latest + args: release --snapshot --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload a Build Artifact - uses: actions/upload-artifact@v4 + if: startsWith(github.ref, 'refs/tags/') + uses: actions/upload-artifact@v7 with: name: gickup path: dist/* diff --git a/.github/workflows/validate-lint.yml b/.github/workflows/validate-lint.yml index fe0815f..0af49b2 100644 --- a/.github/workflows/validate-lint.yml +++ b/.github/workflows/validate-lint.yml @@ -20,18 +20,18 @@ jobs: steps: - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v2 + uses: actions/setup-go@v6 with: go-version: ${{ env.GO_VERSION }} - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v6 with: path: go/src/github.com/cooperspencer/gickup fetch-depth: 0 - name: Cache Go modules - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/go/pkg/mod diff --git a/.golangci.yml b/.golangci.yml index 0ccada8..b6b8f78 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -62,6 +62,14 @@ linters: text: "G704:" - linters: [gosec] text: "G117:" + - linters: [gosec] + text: "G703:" + - linters: [gosec] + path: github/github_test.go + text: "G101:" + - linters: [gosec] + path: radicle/radicle_test.go + text: "G101:" # Global exclusions for staticcheck style suggestions - linters: [staticcheck] text: "SA1019:" @@ -90,6 +98,9 @@ linters: - linters: [revive] path: zip/zip.go text: "var-naming:" + - linters: [revive] + path: zip/zip_test.go + text: "var-naming:" # Per-file existing exclusions - path: main.go text: bad syntax for struct tag pair diff --git a/.goreleaser.yml b/.goreleaser.yml index d865b58..371d759 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,6 +1,7 @@ +version: 2 + builds: - - - goos: + - goos: - freebsd - windows - darwin @@ -9,19 +10,20 @@ builds: - amd64 - arm - arm64 - - 386 + - "386" goarm: - - 6 - - 7 + - "6" + - "7" ignore: - goos: darwin - goarch: 386 + goarch: "386" - goos: windows goarch: arm archives: - - - format: tar.gz + - formats: + - tar.gz format_overrides: - goos: windows - format: zip + formats: + - zip diff --git a/README.md b/README.md index 6daa7a1..5fe1ca0 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ You can clone/mirror repositories from: - Bitbucket - OneDev - Sourcehut +- Opengist - Any You can clone/mirror repositories to: @@ -52,6 +53,7 @@ You can clone/mirror repositories to: - Gogs - OneDev - Sourcehut +- Radicle - Local - S3 @@ -66,6 +68,42 @@ If your hoster is not listed, feel free to open an issue and I will add it. ## How to run the binary version `./gickup path-to-conf.yml` +## Start with the web interface + +```bash +./gickup --webui :8080 /path/to/conf.yml +``` + +Open http://localhost:8080. If the file is missing or empty, the interface opens +configuration setup. Paste your YAML and save; Gickup creates the file and its +parent directory, then loads it within about five seconds. Without a cron schedule, +loading a configuration starts a backup. With a schedule, backups wait for that schedule. +The configuration path defaults to `conf.yml` in the working directory. + +Pass the listen address directly to `--webui`, for example `--webui :8080` +or `--webui 127.0.0.1:9090`. +When `--webui` is supplied, this address takes precedence over `webui.addr` in YAML. +Without the flag, the existing YAML setting still works. +The interface has no authentication; restrict access to trusted users. + +For Docker, mount a **directory** so the interface can create the configuration file: + +```yaml +services: + gickup: + build: . + command: ["--webui", ":8080", "/config/conf.yml"] + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./config:/config + - ./backups:/backups +``` + +Run `mkdir -p config backups` and `docker compose up --build`, then open +http://localhost:8080. The saved configuration persists in `./config/conf.yml`. +Use `/backups` as the local destination path to persist backups on the host. + ## How to run the Docker image ```bash mkdir gickup @@ -96,6 +134,8 @@ https://github.com/cooperspencer/gickup/discussions |Arch|[gickup](https://aur.archlinux.org/packages/gickup/)|[me](https://github.com/cooperspencer)| |Homebrew|[gickup](https://formulae.brew.sh/formula/gickup#default)|| |Fedora|[gickup](https://copr.fedorainfracloud.org/coprs/frostyx/gickup/)|[FrostyX](https://github.com/FrostyX)| +|Scoop|[gickup](https://scoop.sh/#/apps?q=gickup&id=493e856707843828b4491004edfb6ddedc4bd1e1)|| +|Winget|[gickup](https://github.com/microsoft/winget-pkgs/tree/master/manifests/c/cooperspencer/gickup)|| ## Issues The mirroring to Gitlab doesn't work, or at least I can't test it properly because I have no access to a Gitlab EE instance. diff --git a/bitbucket/bitbucket.go b/bitbucket/bitbucket.go index 24a5ff7..47d7811 100644 --- a/bitbucket/bitbucket.go +++ b/bitbucket/bitbucket.go @@ -2,6 +2,7 @@ package bitbucket import ( "net/url" + "os" "time" "github.com/cooperspencer/gickup/logger" @@ -35,7 +36,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { includeorgs := types.GetMap(repo.IncludeOrgs) excludeorgs := types.GetMap(repo.ExcludeOrgs) - client, err := bitbucket.NewBasicAuth(repo.Username, repo.Password) + client, err := bitbucket.NewBasicAuth(repo.Email, repo.Password) if repo.URL == "" { repo.URL = bitbucket.DEFAULT_BITBUCKET_API_BASE_URL @@ -64,7 +65,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { sub.Info(). Msgf("grabbing repositories from %s", repo.User) - repoRes, err := client.Repositories.ListForAccount(&bitbucket.RepositoriesOptions{Owner: repo.User}) + repoRes, err := client.Repositories.ListForAccount(&bitbucket.RepositoriesOptions{Owner: repo.Organization}) var repositories []bitbucket.Repository if err == nil { repositories = repoRes.Items @@ -95,6 +96,9 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } } } + } else { + sub.Error().Msgf("Exiting due to error fetching Bitbucket workspace repositories: %s", err.Error()) + os.Exit(1) } for _, r := range repositories { @@ -125,13 +129,22 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } + if exclude[r.Name] { + continue + } + + origin := repo + if repo.Username != "" { + origin.User = repo.Username + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, URL: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string), SSHURL: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string), Token: repo.Token, - Origin: repo, + Origin: origin, Owner: user, Hoster: types.GetHost(repo.URL), Description: r.Description, @@ -141,17 +154,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, URL: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string), SSHURL: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string), Token: repo.Token, - Origin: repo, + Origin: origin, Owner: user, Hoster: types.GetHost(repo.URL), Description: r.Description, diff --git a/conf.example.yml b/conf.example.yml index 274ec36..0f8ecc9 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -7,6 +7,7 @@ source: # token_file: token.txt user: some-user # the user you want to clone the repositories from. # if you want to get everything from your user, leave out the user parameter and just use the token. + # url: https://example.ghe.com # optional: GitHub Enterprise Server base URL; defaults to https://github.com # for the clone process, either use: # - username + password # - sshkey @@ -29,6 +30,7 @@ source: wiki: true # includes wiki too issues: true # back up issues, works only locally starred: true # includes the user's starred repositories too + contributed: true # includes repositories the user contributed to filter: stars: 100 # only clone repos with 100 stars lastactivity: 1y # only clone repos which had activity during the last year @@ -38,6 +40,13 @@ source: - java excludeforks: true # exclude forked repositories gists: true # clone gists too + # alternatively, authenticate with a GitHub App: + # - app_id: 123456 # GitHub App ID (numeric) + # app_installation_id: 78901234 # Installation ID of the App on the target account + # app_private_key_file: /path/to/private.pem # path to the App's RSA private key PEM file + # user: some-user # user or org whose repos to back up (optional if App is self-installed) + # organization: some-org # alternatively back up an organization's repos + # ssh: false # note: 'contributed' is not supported with App auth gitea: - token: some-token # token_file: token.txt # alternatively, specify token in a file @@ -143,9 +152,11 @@ source: excludeforks: true # exclude forked repositories bitbucket: - user: some-user # the user you want to clone the repositories from. - - token: some-token + token: some-token # token_file: token.txt # alternatively, specify token in a file url: http(s)://url-to-bitbucket # if empty, it uses https://bitbucket.org + organization: some-workspace # workspace to list repositories from + email: your-email@example.com # email is used for Bitbucket API basic auth username: your-user # user is used to clone the repo with password: your-password ssh: true # can be true or false @@ -201,6 +212,19 @@ source: wiki: true # includes wiki too filter: lastactivity: 1y # only clone repos which had activity during the last year + opengist: + - url: http(s)://url-to-opengist # required, base url of your Opengist instance, e.g. https://gist.example.com + token: some-token # Opengist personal access token (og_...); used for the API and to clone private gists + # token_file: token.txt # alternatively, specify token in a file + # user: some-user # optional: back up this user's gists (GET /api/users//gists). + # leave user out and set a token to back up the token owner's own gists (GET /api/gists). + # leave out both user and token to back up only public gists (GET /api/gists/public). + # note: Opengist's git-over-http must be enabled on the server to clone (OG_HTTP_GIT_ENABLED=true). + exclude: # this excludes the gists with these slugs/ids + - foo + - bar + include: # this only includes the gists with these slugs/ids + - foobar any: - url: url-to-any-repo # can be https, http or ssh user: your-preferred-user # the user to want to associate with this repo, default: git @@ -238,6 +262,10 @@ destination: - token: some-token # token_file: token.txt # alternatively, specify token in a file url: http(s)://url-to-gitlab + user: some-name # can be a user or a group, it must exist on the system + visibility: + repositories: private # private, public, default: source repository visibility + force: false # force push to destination mirror: enabled: true # if set to true, gickup will clone the repository and push it to gitlab itself github: @@ -247,6 +275,16 @@ destination: visibility: repositories: private # private, public, default: private force: false # force push to destination + # alternatively, authenticate with a GitHub App: + # requires Contents: write, plus Administration: write to create repositories + # - app_id: 123456 # GitHub App ID (numeric) + # app_installation_id: 78901234 # installation on the destination organization, including for enterprise-owned Apps + # app_private_key_file: /path/to/private.pem # path to the App's RSA private key PEM file + # organization: whatever # destination organization; required for App authentication + # url: https://github.example.com # GitHub Enterprise Server base URL; omit for github.com + # visibility: + # repositories: private + # force: false onedev: - token: some-token # token_file: token.txt # alternatively, specify token in a file @@ -271,18 +309,29 @@ destination: bare: true # clone the repositories as bare mirror: true # create mirror clones lfs: true # clone lfs repos, !! ATTENTION !! git and git-lfs must be installed on the system! + radicle: + # rad reads the environment itself: RAD_HOME selects which radicle profile to + # mirror into (defaults to ~/.radicle), and RAD_PASSPHRASE unlocks its key if + # it has a passphrase. Export them before running gickup. + - force: true # if set to true, refs that diverged from upstream are overwritten (non-fast-forward updates) + prune: true # if set to true, refs that no longer exist upstream are deleted from the mirror + visibility: source # public, private or source, source follows the visibility of the source repository. default: source + issues: false # [COMING SOON] recreate the source repo's issues as radicle issues (the source must also have issues: true). s3: - endpoint: somewhere:9000 # whatever your s3 endpoint is structured: true # checks repos out like hostersite/user|organization/repo bucket: your-bucket-name + use_static_creds: true # if true, use static credentials (accesskey/secretkey/token); if false, use IAM instance credentials (e.g. for AWS EC2/ECS) accesskey: your-access-key # can be an environment variable, just don't add a $ in front of it secretkey: your-secret-key # can be an environment variable, just don't add a $ in front of it token: your-token # can be an environment variable, just don't add a $ in front of it + region: your-region # S3 region, for example us-east-1 zip: false # if true, will zip the entire git repo into a single zip file and upload that instead usessl: true # wheter to use ssl or not storageclass: "" # storage class for all repos to be uploaded to this S3 bucket. E.g. for AWS: STANDARD, STANDARD_IA, GLACIER, etc. src_repo_url_tag_key: "" # if set, tag each uploaded object with this key and the source repo URL as value. Leave unset to disable tagging. datecreatedir: false # if true, gickup will create a directory for backup with the creation date + lfs: true # clone lfs repos, !! ATTENTION !! git and git-lfs must be installed on the system! azureblob: - url: https://yourstorageaccount.blob.core.windows.net # blob storage endpoint for your storage account, can be a SAS url container: name of your blob container @@ -293,6 +342,17 @@ destination: structured: true # checks repos out like hostersite/user|organization/repo zip: false # if true, will zip the entire git repo into a single zip file and upload that instead datecreatedir: false # if true, gickup will create a directory for backup with the creation date + webdav: + - url: https://webdav.example.com/dav # the WebDAV endpoint to upload the backups to (e.g. Nextcloud, Apache mod_dav, rclone serve webdav) + username: your-username # HTTP Basic auth username + password: your-password # HTTP Basic auth password + path: repos # subdirectory on the WebDAV server to store the backups under (repos end up under url/path/) + structured: true # checks repos out like hostersite/user|organization/repo + zip: false # if true, will zip the entire git repo into a single zip file and upload that instead + datecreatedir: false # if true, gickup will create a directory for backup with the creation date +webui: # optional - enables the built-in web interface + addr: ":8282" # address and port to listen on + cron: 0 22 * * * # optional - when cron is not provided, the program runs once and exits. # Otherwise, it runs according to the cron schedule. # See timezone commentary in docker-compose.yml for making sure this container runs @@ -315,6 +375,8 @@ metrics: urls: - http(s)://url-to-make-request-to - http(s)://another-url-to-make-request-to + failure_urls: # optional - if the backup run encountered any errors, these URLs are pinged instead of the ones above. Useful with services like healthchecks.io that expose a dedicated failure endpoint (e.g. https://hc-ping.com/uuid/fail) + - http(s)://url-to-ping-on-failure push: ntfy: - url: http(s)://url-to-ntfy/your-topic diff --git a/gickup_spec.json b/gickup_spec.json index 41e8c1c..59b42ce 100644 --- a/gickup_spec.json +++ b/gickup_spec.json @@ -15,6 +15,17 @@ "metrics": { "$ref": "#/definitions/metrics" }, + "webui": { + "type": "object", + "description": "Configuration for the built-in web interface", + "additionalProperties": false, + "properties": { + "addr": { + "type": "string", + "description": "Address and port to listen on, e.g. ':8282'" + } + } + }, "source": { "type": "object", "description": "Configuration for all git hoster source endpoints", @@ -70,6 +81,18 @@ "gists": { "$ref": "#/definitions/source/properties/gists" }, + "contributed": { + "$ref": "#/definitions/source/properties/contributed" + }, + "app_id": { + "$ref": "#/definitions/source/properties/app_id" + }, + "app_installation_id": { + "$ref": "#/definitions/source/properties/app_installation_id" + }, + "app_private_key_file": { + "$ref": "#/definitions/source/properties/app_private_key_file" + }, "filter": { "$ref": "#/definitions/filter" } @@ -394,6 +417,12 @@ "user": { "$ref": "#/definitions/source/properties/user" }, + "organization": { + "$ref": "#/definitions/source/properties/organization" + }, + "email": { + "$ref": "#/definitions/source/properties/email" + }, "username": { "$ref": "#/definitions/source/properties/username" }, @@ -431,6 +460,36 @@ "additionalProperties": false } }, + "opengist": { + "type": "array", + "items": { + "type": "object", + "properties": { + "token": { + "$ref": "#/definitions/source/properties/token" + }, + "token_file": { + "$ref": "#/definitions/source/properties/token_file" + }, + "url": { + "$ref": "#/definitions/source/properties/url" + }, + "user": { + "$ref": "#/definitions/source/properties/user" + }, + "exclude": { + "$ref": "#/definitions/source/properties/exclude" + }, + "include": { + "$ref": "#/definitions/source/properties/include" + } + }, + "additionalProperties": false + }, + "required": [ + "url" + ] + }, "any": { "type": "array", "items": { @@ -531,6 +590,10 @@ "type": "string", "description": "The bucket to store the backups in" }, + "use_static_creds": { + "type": "boolean", + "description": "If true, use static credentials (accesskey/secretkey/token). If false, use IAM instance credentials (e.g. for AWS EC2/ECS). Default: false" + }, "accesskey": { "type": "string", "description": "The access key to authenticate against the S3 server" @@ -547,6 +610,10 @@ "type": "string", "description": "The token to authenticate against the S3 server" }, + "region": { + "type": "string", + "description": "The S3 region to use" + }, "zip": { "type": "boolean", "description": "If true, zip the entire Git repo into a single zip file and upload that instead" @@ -562,6 +629,10 @@ "datecreatedir": { "type": "boolean", "description": "If true, gickup will create a directory for backup with the creation date" + }, + "lfs": { + "type": "boolean", + "description": "uses lfs to clone repositories" } }, "additionalProperties": false @@ -584,6 +655,15 @@ "force": { "$ref": "#/definitions/destination/properties/force" }, + "app_id": { + "$ref": "#/definitions/destination/properties/app_id" + }, + "app_installation_id": { + "$ref": "#/definitions/destination/properties/app_installation_id" + }, + "app_private_key_file": { + "$ref": "#/definitions/destination/properties/app_private_key_file" + }, "visibility": { "type": "object", "properties": { @@ -611,6 +691,9 @@ "url": { "$ref": "#/definitions/destination/properties/url" }, + "user": { + "$ref": "#/definitions/destination/properties/user" + }, "force": { "$ref": "#/definitions/destination/properties/force" }, @@ -622,6 +705,15 @@ } }, "additionalProperties": false + }, + "visibility": { + "type": "object", + "properties": { + "repositories": { + "$ref": "#/definitions/visibility/properties/repositories" + } + }, + "additionalProperties": false } }, "additionalProperties": false @@ -767,6 +859,120 @@ }, "additionalProperties": false } + }, + "azureblob": { + "type": "array", + "items": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The Azure Blob Storage endpoint URL, optionally including a SAS token" + }, + "container": { + "type": "string", + "description": "The Azure Blob Storage container to store the backups in" + }, + "useclicredential": { + "type": "boolean", + "description": "If true, use Azure CLI credentials to authenticate" + }, + "tenantid": { + "type": "string", + "description": "The Azure tenant ID for client secret authentication" + }, + "clientid": { + "type": "string", + "description": "The Azure app registration client ID" + }, + "clientsecret": { + "type": "string", + "description": "The Azure app registration client secret" + }, + "structured": { + "type": "boolean", + "description": "if set to `true`, it checks out the repos in a more structured way, like `hoster/user|organization/repository`" + }, + "zip": { + "type": "boolean", + "description": "If true, zip the entire Git repo into a single zip file and upload that instead" + }, + "datecreatedir": { + "type": "boolean", + "description": "If true, gickup will create a directory for backup with the creation date" + } + }, + "additionalProperties": false + } + }, + "webdav": { + "type": "array", + "description": "Upload bare-cloned repositories to a WebDAV server using HTTP Basic auth. Stale remote files are pruned via PROPFIND/DELETE.", + "items": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The WebDAV endpoint URL to upload the backups to (e.g. a Nextcloud, Apache mod_dav or rclone serve webdav server)" + }, + "username": { + "type": "string", + "description": "HTTP Basic auth username" + }, + "password": { + "type": "string", + "description": "HTTP Basic auth password" + }, + "path": { + "type": "string", + "description": "Subdirectory on the WebDAV server to store the backups under; repositories end up under url/path/" + }, + "structured": { + "type": "boolean", + "description": "if set to `true`, it checks out the repos in a more structured way, like `hoster/user|organization/repository`" + }, + "zip": { + "type": "boolean", + "description": "If true, zip the entire Git repo into a single zip file and upload that instead" + }, + "datecreatedir": { + "type": "boolean", + "description": "If true, gickup will create a directory for backup with the creation date" + } + }, + "additionalProperties": false + } + }, + "radicle": { + "type": "array", + "description": "Mirror repositories into the storage of a local Radicle node. Requires the `git`, `rad` and `git-remote-rad` binaries.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "force": { + "type": "boolean", + "description": "overwrite refs on the mirror that have diverged from upstream (non-fast-forward updates)" + }, + "prune": { + "type": "boolean", + "description": "delete refs on the mirror that no longer exist upstream" + }, + "visibility": { + "type": "string", + "enum": [ + "public", + "private", + "source" + ], + "description": "visibility of the mirrored repository, `source` follows the source repository. default: source" + }, + "issues": { + "type": "boolean", + "description": "[COMING SOON] recreate the source repository's issues." + } + } + } } } } @@ -801,6 +1007,11 @@ "type": "string", "description": "The organization to clone the repositories from" }, + "email": { + "$id": "#/definitions/source/properties/email", + "type": "string", + "description": "The email address to authenticate against the source" + }, "username": { "$id": "#/definitions/source/properties/username", "type": "string", @@ -876,6 +1087,26 @@ "$id": "#/definitions/source/properties/gists", "type": "boolean", "description": "Include the gists in the backup" + }, + "contributed": { + "$id": "#/definitions/source/properties/contributed", + "type": "boolean", + "description": "Include repositories the user contributed to" + }, + "app_id": { + "$id": "#/definitions/source/properties/app_id", + "type": "integer", + "description": "GitHub App ID (numeric). Used together with app_installation_id and app_private_key_file to authenticate as a GitHub App." + }, + "app_installation_id": { + "$id": "#/definitions/source/properties/app_installation_id", + "type": "integer", + "description": "Installation ID of the GitHub App on the target account or organization." + }, + "app_private_key_file": { + "$id": "#/definitions/source/properties/app_private_key_file", + "type": "string", + "description": "Path to the RSA private key PEM file for the GitHub App." } }, "additionalProperties": false @@ -928,6 +1159,21 @@ "$id": "#/definitions/destination/properties/force", "type": "boolean", "description": "Force push the repositories" + }, + "app_id": { + "$id": "#/definitions/destination/properties/app_id", + "type": "integer", + "description": "GitHub App ID (numeric). Used together with app_installation_id and app_private_key_file to authenticate as a GitHub App." + }, + "app_installation_id": { + "$id": "#/definitions/destination/properties/app_installation_id", + "type": "integer", + "description": "Installation ID of the GitHub App on the target account or organization." + }, + "app_private_key_file": { + "$id": "#/definitions/destination/properties/app_private_key_file", + "type": "string", + "description": "Path to the RSA private key PEM file for the GitHub App." } }, "additionalProperties": false diff --git a/gitcmd/gitcmd.go b/gitcmd/gitcmd.go index 3169c2e..7283bb3 100644 --- a/gitcmd/gitcmd.go +++ b/gitcmd/gitcmd.go @@ -9,6 +9,36 @@ import ( "strings" ) +type Auth struct { + Username string + Password string +} + +// envAuthHelper answers git's credential "get" request from the environment, +// so credentials never end up in URLs, command arguments or git config files. +const envAuthHelper = `!f() { test "$1" = get || return 0; echo "username=${GICKUP_GIT_USERNAME}"; echo "password=${GICKUP_GIT_PASSWORD}"; }; f` + +// Env passes the credentials through a credential helper instead of +// http.extraHeader. git lfs adds extra headers to object downloads as well, +// where the LFS server already provides its own Authorization header, and +// servers like gitlab.com reject the duplicate header with 400 Bad Request. +func (a *Auth) Env() []string { + if a == nil || (a.Username == "" && a.Password == "") { + return nil + } + return []string{ + "GICKUP_GIT_USERNAME=" + a.Username, + "GICKUP_GIT_PASSWORD=" + a.Password, + "GIT_CONFIG_COUNT=2", + // The empty value resets helpers from system or global config, + // so no other helper stores these credentials. + "GIT_CONFIG_KEY_0=credential.helper", + "GIT_CONFIG_VALUE_0=", + "GIT_CONFIG_KEY_1=credential.helper", + "GIT_CONFIG_VALUE_1=" + envAuthHelper, + } +} + type GitCmd struct { CMD string } @@ -23,55 +53,64 @@ func New() (GitCmd, error) { return GitCmd{CMD: "git"}, nil } -func (g GitCmd) Clone(url, reponame string, bare bool, mirror bool) error { - cmd := exec.CommandContext(context.Background(), g.CMD, "clone", url, reponame) +func (g GitCmd) Command(ctx context.Context, auth *Auth, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, g.CMD, args...) + if authEnv := auth.Env(); len(authEnv) > 0 { + cmd.Env = append(os.Environ(), authEnv...) + } + return cmd +} + +func (g GitCmd) Clone(url, reponame string, bare bool, mirror bool, auth *Auth) error { + args := []string{"clone", url, reponame} if bare { - cmd.Args = append(cmd.Args, "--bare") + args = append(args, "--bare") } if mirror { - cmd.Args = append(cmd.Args, "--mirror") + args = append(args, "--mirror") } + cmd := g.Command(context.Background(), auth, args...) return cmd.Run() } -func (g GitCmd) Pull(bare bool, mirror bool, repopath string) error { +func (g GitCmd) Pull(bare bool, mirror bool, repopath string, auth *Auth) error { var args []string if bare || mirror { args = []string{"-C", repopath, "fetch", "--all"} } else { args = []string{"-C", repopath, "pull", "--all"} } - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), auth, args...) return cmd.Run() } -func (g GitCmd) Fetch(path string) error { +func (g GitCmd) Fetch(path string, auth *Auth) error { _, err := os.Stat(path) if err != nil { return err } args := []string{"-C", path, "fetch", "--all", "--tags"} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), auth, args...) return cmd.Run() } -func (g GitCmd) LFSFetch(path string) error { +func (g GitCmd) LFSFetch(path string, auth *Auth) error { _, err := os.Stat(path) if err != nil { return err } args := []string{"-C", path, "lfs", "fetch", "--all"} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), auth, args...) return cmd.Run() } -func (g GitCmd) MirrorPull(path string) error { +func (g GitCmd) MirrorPull(path string, auth *Auth) error { _, err := os.Stat(path) if err != nil { return err } args := []string{"-C", path, "pull", "--all", "--tags"} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), auth, args...) return cmd.Run() } @@ -81,18 +120,18 @@ func (g GitCmd) NewRemote(name, url, path string) error { return err } args := []string{"-C", path, "remote", "add", name, url} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), nil, args...) return cmd.Run() } -func (g GitCmd) Push(path, remote string) error { +func (g GitCmd) Push(path, remote string, auth *Auth) error { _, err := os.Stat(path) if err != nil { return err } args := []string{"-C", path, "push", "--all", remote} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), auth, args...) output, err := cmd.CombinedOutput() if err != nil { @@ -110,8 +149,8 @@ func (g GitCmd) Checkout(path, branch string) error { if err != nil { return err } - args := []string{"checkout", branch} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + args := []string{"-C", path, "checkout", branch} + cmd := g.Command(context.Background(), nil, args...) output, err := cmd.CombinedOutput() if err != nil { @@ -125,10 +164,21 @@ func (g GitCmd) Checkout(path, branch string) error { } func (g GitCmd) SSHPush(path, remote, key string) error { - err := os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -i %s", key)) + _, err := os.Stat(path) if err != nil { return err } + args := []string{"-C", path, "push", "--all", remote} + cmd := g.Command(context.Background(), nil, args...) + cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=ssh -i %s", key)) + + output, err := cmd.CombinedOutput() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return fmt.Errorf("%s", strings.TrimSuffix(string(output), "\n")) + } + } - return g.Push(path, remote) + return err } diff --git a/gitcmd/gitcmd_test.go b/gitcmd/gitcmd_test.go new file mode 100644 index 0000000..75f86ec --- /dev/null +++ b/gitcmd/gitcmd_test.go @@ -0,0 +1,292 @@ +package gitcmd + +import ( + "context" + "reflect" + "strings" + "testing" +) + +func expectedAuthEnv(username, password string) []string { + return []string{ + "GICKUP_GIT_USERNAME=" + username, + "GICKUP_GIT_PASSWORD=" + password, + "GIT_CONFIG_COUNT=2", + "GIT_CONFIG_KEY_0=credential.helper", + "GIT_CONFIG_VALUE_0=", + "GIT_CONFIG_KEY_1=credential.helper", + "GIT_CONFIG_VALUE_1=" + envAuthHelper, + } +} + +func TestAuth_Env_Nil(t *testing.T) { + t.Parallel() + + var auth *Auth + if got := auth.Env(); got != nil { + t.Fatalf("expected nil env for nil Auth, got %v", got) + } +} + +func TestAuth_Env_Empty(t *testing.T) { + t.Parallel() + + auth := &Auth{Username: "", Password: ""} + if got := auth.Env(); got != nil { + t.Fatalf("expected nil env for empty Auth, got %v", got) + } +} + +func TestAuth_Env_GitHubApp(t *testing.T) { + t.Parallel() + + auth := &Auth{ + Username: "x-access-token", + Password: "test-token", + } + + env := auth.Env() + expectedEnv := expectedAuthEnv("x-access-token", "test-token") + + if !reflect.DeepEqual(env, expectedEnv) { + t.Fatalf("auth.Env() = %v, want %v", env, expectedEnv) + } +} + +func TestAuth_Env_UserPassword(t *testing.T) { + t.Parallel() + + auth := &Auth{ + Username: "octocat", + Password: "secretpassword456", + } + + env := auth.Env() + expectedEnv := expectedAuthEnv("octocat", "secretpassword456") + + if !reflect.DeepEqual(env, expectedEnv) { + t.Fatalf("auth.Env() = %v, want %v", env, expectedEnv) + } +} + +func TestGitCmd_Command_WithoutAuth(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + cmd := g.Command(context.Background(), nil, "status") + + if cmd.Path != "git" && !strings.HasSuffix(cmd.Path, "git") && !strings.HasSuffix(cmd.Path, "git.exe") { + t.Errorf("cmd.Path = %q, want 'git'", cmd.Path) + } + if !reflect.DeepEqual(cmd.Args, []string{"git", "status"}) { + t.Errorf("cmd.Args = %v, want ['git', 'status']", cmd.Args) + } + if cmd.Env != nil { + t.Errorf("cmd.Env for nil Auth = %v, want nil", cmd.Env) + } +} + +func TestGitCmd_Command_WithAuth(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + auth := &Auth{Username: "x-access-token", Password: "test-token"} + cmd := g.Command(context.Background(), auth, "clone", "https://github.com/owner/repo.git", "/tmp/repo") + + if cmd.Env == nil { + t.Fatal("cmd.Env is nil, want environment variables with auth config") + } + + present := make(map[string]bool, len(cmd.Env)) + for _, e := range cmd.Env { + present[e] = true + } + for _, want := range expectedAuthEnv("x-access-token", "test-token") { + if !present[want] { + t.Errorf("cmd.Env missing %q, env = %v", want, cmd.Env) + } + } + + // Verify the URL in cmd.Args does NOT contain token + for _, arg := range cmd.Args { + if strings.Contains(arg, "test-token") { + t.Errorf("token leaked into command arguments: %q", arg) + } + } +} + +func TestGitCmd_CloneCommand(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + url string + reponame string + bare bool + mirror bool + auth *Auth + wantArgs []string + wantAuth bool + }{ + { + name: "regular clone without auth", + url: "https://github.com/owner/repo.git", + reponame: "/tmp/repo", + bare: false, + mirror: false, + auth: nil, + wantArgs: []string{"git", "clone", "https://github.com/owner/repo.git", "/tmp/repo"}, + wantAuth: false, + }, + { + name: "bare clone with token", + url: "https://github.com/owner/repo.git", + reponame: "/tmp/repo.git", + bare: true, + mirror: false, + auth: &Auth{Username: "x-access-token", Password: "my-token"}, + wantArgs: []string{"git", "clone", "https://github.com/owner/repo.git", "/tmp/repo.git", "--bare"}, + wantAuth: true, + }, + { + name: "mirror clone with user/pass", + url: "https://github.com/owner/repo.git", + reponame: "/tmp/repo.git", + bare: true, + mirror: true, + auth: &Auth{Username: "user", Password: "pass"}, + wantArgs: []string{"git", "clone", "https://github.com/owner/repo.git", "/tmp/repo.git", "--bare", "--mirror"}, + wantAuth: true, + }, + } + + g := GitCmd{CMD: "git"} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + args := []string{"clone", tt.url, tt.reponame} + if tt.bare { + args = append(args, "--bare") + } + if tt.mirror { + args = append(args, "--mirror") + } + cmd := g.Command(context.Background(), tt.auth, args...) + + if !reflect.DeepEqual(cmd.Args, tt.wantArgs) { + t.Errorf("cmd.Args = %v, want %v", cmd.Args, tt.wantArgs) + } + + if tt.wantAuth && cmd.Env == nil { + t.Error("cmd.Env is nil, want auth environment") + } + if !tt.wantAuth && cmd.Env != nil { + t.Errorf("cmd.Env = %v, want nil", cmd.Env) + } + + for _, arg := range cmd.Args { + if strings.Contains(arg, "@") { + t.Errorf("URL in arguments contains credentials: %s", arg) + } + } + }) + } +} + +func TestGitCmd_PullCommand(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + auth := &Auth{Username: "x-access-token", Password: "test-token"} + + // Bare/mirror fetch + bareArgs := []string{"-C", "/tmp/repo.git", "fetch", "--all"} + bareCmd := g.Command(context.Background(), auth, bareArgs...) + wantBareArgs := []string{"git", "-C", "/tmp/repo.git", "fetch", "--all"} + if !reflect.DeepEqual(bareCmd.Args, wantBareArgs) { + t.Errorf("bareCmd.Args = %v, want %v", bareCmd.Args, wantBareArgs) + } + if bareCmd.Env == nil { + t.Error("bareCmd.Env is nil, want auth environment") + } + + // Non-bare pull + pullArgs := []string{"-C", "/tmp/repo", "pull", "--all"} + pullCmd := g.Command(context.Background(), auth, pullArgs...) + wantPullArgs := []string{"git", "-C", "/tmp/repo", "pull", "--all"} + if !reflect.DeepEqual(pullCmd.Args, wantPullArgs) { + t.Errorf("pullCmd.Args = %v, want %v", pullCmd.Args, wantPullArgs) + } + if pullCmd.Env == nil { + t.Error("pullCmd.Env is nil, want auth environment") + } +} + +func TestGitCmd_LFSFetchCommand(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + auth := &Auth{Username: "x-access-token", Password: "test-token"} + + lfsArgs := []string{"-C", "/tmp/repo.git", "lfs", "fetch", "--all"} + cmd := g.Command(context.Background(), auth, lfsArgs...) + wantArgs := []string{"git", "-C", "/tmp/repo.git", "lfs", "fetch", "--all"} + + if !reflect.DeepEqual(cmd.Args, wantArgs) { + t.Errorf("cmd.Args = %v, want %v", cmd.Args, wantArgs) + } + if cmd.Env == nil { + t.Fatal("cmd.Env is nil, want auth environment for LFS fetch") + } + + foundHelper := false + for _, e := range cmd.Env { + if e == "GIT_CONFIG_VALUE_1="+envAuthHelper { + foundHelper = true + } + // git lfs would add an extra header to object downloads next to the + // Authorization header from the LFS batch response. + if strings.Contains(e, "http.extraHeader") { + t.Errorf("cmd.Env sets http.extraHeader, which breaks LFS object downloads: %q", e) + } + } + if !foundHelper { + t.Error("cmd.Env does not contain the credential helper for LFS fetch") + } +} + +func TestGitCmd_CheckoutCommand(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + + // Checkout must scope the command to the repo path with -C, otherwise + // git runs against the process's cwd instead of the cloned repo. + checkoutArgs := []string{"-C", "/tmp/repo", "checkout", "main"} + cmd := g.Command(context.Background(), nil, checkoutArgs...) + wantArgs := []string{"git", "-C", "/tmp/repo", "checkout", "main"} + + if !reflect.DeepEqual(cmd.Args, wantArgs) { + t.Errorf("cmd.Args = %v, want %v", cmd.Args, wantArgs) + } +} + +func TestGitCmd_LocalCommandsHaveNoAuth(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + + // Checkout should have no auth + checkoutCmd := g.Command(context.Background(), nil, "checkout", "main") + if checkoutCmd.Env != nil { + t.Errorf("checkoutCmd.Env = %v, want nil", checkoutCmd.Env) + } + + // NewRemote should have no auth + remoteCmd := g.Command(context.Background(), nil, "-C", "/tmp/repo", "remote", "add", "origin", "https://github.com/owner/repo.git") + if remoteCmd.Env != nil { + t.Errorf("remoteCmd.Env = %v, want nil", remoteCmd.Env) + } +} diff --git a/gitea/gitea.go b/gitea/gitea.go index 620c519..19a1933 100644 --- a/gitea/gitea.go +++ b/gitea/gitea.go @@ -322,6 +322,11 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { continue } + + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -354,10 +359,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(repo.Include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, @@ -468,6 +469,11 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { continue } + + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -500,10 +506,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(repo.Include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, diff --git a/github/github.go b/github/github.go index b7c4d9e..cc412ad 100644 --- a/github/github.go +++ b/github/github.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/bradleyfalzon/ghinstallation/v2" "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" "github.com/google/go-github/v74/github" @@ -47,13 +48,102 @@ type V4Repo struct { var sub zerolog.Logger -func getv4(token, user string) []V4Repo { +// githubInstanceURL returns the canonical base URL for a GitHub instance. +// Defaults to "https://github.com" if rawURL is empty. +func githubInstanceURL(rawURL string) string { + if rawURL == "" { + return "https://github.com" + } + return strings.TrimRight(rawURL, "/") +} + +// isGHE returns true when the URL refers to a GitHub Enterprise Server instance. +func isGHE(instanceURL string) bool { + u := strings.TrimPrefix(instanceURL, "https://") + u = strings.TrimPrefix(u, "http://") + return !strings.HasPrefix(u, "github.com") +} + +// hosterFromURL extracts the hostname portion from a GitHub instance URL. +func hosterFromURL(instanceURL string) string { + u := strings.TrimPrefix(instanceURL, "https://") + u = strings.TrimPrefix(u, "http://") + if idx := strings.Index(u, "/"); idx != -1 { + return u[:idx] + } + return u +} + +// newGithubClient creates an authenticated GitHub client. +// When App authentication fields are configured it uses a GitHub App installation token; +// otherwise it falls back to a personal access token or an unauthenticated client. +// It returns the client, the access token string (for git operations), and any error. +func newGithubClient(ctx context.Context, repo types.GenRepo) (*github.Client, string, error) { + instURL := githubInstanceURL(repo.URL) + + if repo.HasAppAuth() { + itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, repo.AppID, repo.AppInstallationID, repo.AppPrivateKeyFile) + if err != nil { + return nil, "", fmt.Errorf("github app auth: %w", err) + } + var client *github.Client + if isGHE(instURL) { + apiBase := instURL + "/api/v3/" + uploadURL := instURL + "/uploads/" + itr.BaseURL = apiBase + client, err = github.NewEnterpriseClient(apiBase, uploadURL, &http.Client{Transport: itr}) + if err != nil { + return nil, "", fmt.Errorf("github enterprise client: %w", err) + } + } else { + client = github.NewClient(&http.Client{Transport: itr}) + } + token, err := itr.Token(ctx) + if err != nil { + return nil, "", fmt.Errorf("github app token: %w", err) + } + return client, token, nil + } + + token := repo.GetToken() + + var tc *http.Client + if token != "" { + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) + tc = oauth2.NewClient(ctx, ts) + } + + if isGHE(instURL) { + apiBase := instURL + "/api/v3/" + uploadURL := instURL + "/uploads/" + client, err := github.NewEnterpriseClient(apiBase, uploadURL, tc) + if err != nil { + return nil, "", fmt.Errorf("github enterprise client: %w", err) + } + return client, token, nil + } + + if token == "" { + return github.NewClient(nil), "", nil + } + + return github.NewClient(tc), token, nil +} + +func getv4(token, user, instanceURL string) []V4Repo { repos := []V4Repo{} tokenSource := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, ) oauth2Client := oauth2.NewClient(context.Background(), tokenSource) - client := githubv4.NewClient(oauth2Client) + + var client *githubv4.Client + if isGHE(instanceURL) { + graphqlURL := instanceURL + "/api/graphql" + client = githubv4.NewEnterpriseClient(graphqlURL, oauth2Client) + } else { + client = githubv4.NewClient(oauth2Client) + } var query Query variables := map[string]interface{}{ @@ -81,7 +171,7 @@ func getv4(token, user string) []V4Repo { return repos } -func addWiki(r github.Repository, repo types.GenRepo, token string) types.Repo { +func addWiki(r github.Repository, repo types.GenRepo, token, hoster string) types.Repo { if !(r.GetHasWiki() && repo.Wiki && types.StatRemote(r.GetCloneURL(), r.GetSSHURL(), repo)) { return types.Repo{} @@ -94,7 +184,7 @@ func addWiki(r github.Repository, repo types.GenRepo, token string) types.Repo { Token: token, Origin: repo, Owner: r.GetOwner().GetLogin(), - Hoster: "github.com", + Hoster: hoster, Description: r.GetDescription(), Private: r.GetPrivate(), } @@ -105,7 +195,9 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { ran := false repos := []types.Repo{} for _, repo := range conf.Source.Github { - sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com") + instURL := githubInstanceURL(repo.URL) + hoster := hosterFromURL(instURL) + sub = logger.CreateSubLogger("stage", "github", "url", instURL) err := repo.Filter.ParseDuration() if err != nil { sub.Warn(). @@ -128,22 +220,15 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { i := 1 githubrepos := []*github.Repository{} - token := repo.GetToken() - var client *github.Client - if token == "" { - client = github.NewClient(nil) - } else { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(context.TODO(), ts) - - client = github.NewClient(tc) + client, token, err := newGithubClient(context.TODO(), repo) + if err != nil { + sub.Error().Msg(err.Error()) + continue } v4user := repo.User - if token != "" { + if token != "" && !repo.HasAppAuth() { user, _, err := client.Users.Get(context.TODO(), "") if err != nil { sub.Error(). @@ -158,20 +243,38 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } if token != "" && v4user != "" && repo.Contributed { - for _, r := range getv4(token, v4user) { - githubRepo, _, err := client.Repositories.Get(context.Background(), r.User, r.Repository) - if err != nil { - sub.Error(). - Msg(err.Error()) - continue + if repo.HasAppAuth() { + sub.Warn().Msg("contributed repos are not supported with GitHub App authentication, skipping") + } else { + for _, r := range getv4(token, v4user, instURL) { + githubRepo, _, err := client.Repositories.Get(context.Background(), r.User, r.Repository) + if err != nil { + sub.Error(). + Msg(err.Error()) + continue + } + githubrepos = append(githubrepos, githubRepo) } - githubrepos = append(githubrepos, githubRepo) } } for { opt.Page = i - repos, status, err := client.Repositories.List(context.TODO(), repo.User, opt) + var fetchedRepos []*github.Repository + var status *github.Response + var err error + + if repo.HasAppAuth() { + appListOpt := &github.ListOptions{Page: i, PerPage: opt.PerPage} + var result *github.ListRepositories + result, status, err = client.Apps.ListRepos(context.TODO(), appListOpt) + if result != nil { + fetchedRepos = result.Repositories + } + } else { + fetchedRepos, status, err = client.Repositories.List(context.TODO(), repo.User, opt) + } + if err != nil { sub.Error(). Msg(err.Error()) @@ -188,14 +291,15 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if status.StatusCode == http.StatusNotFound { + if status != nil && status.StatusCode == http.StatusNotFound { break } + break } - if len(repos) == 0 { + if len(fetchedRepos) == 0 { break } - githubrepos = append(githubrepos, repos...) + githubrepos = append(githubrepos, fetchedRepos...) i++ } @@ -264,6 +368,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } + if exclude[*r.Name] { + continue + } + + if excludeorgs[r.GetOwner().GetLogin()] { + continue + } + if include[*r.Name] { repos = append(repos, types.Repo{ Name: r.GetName(), @@ -272,13 +384,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Token: token, Origin: repo, Owner: r.GetOwner().GetLogin(), - Hoster: "github.com", + Hoster: hoster, Description: r.GetDescription(), Private: r.GetPrivate(), Issues: GetIssues(r, client, repo), NoTokenUser: true, }) - wiki := addWiki(*r, repo, token) + wiki := addWiki(*r, repo, token, hoster) if wiki.Name != "" { repos = append(repos, wiki) } @@ -286,14 +398,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[*r.Name] { - continue - } - - if excludeorgs[r.GetOwner().GetLogin()] { - continue - } - if len(repo.Include) == 0 { if len(includeorgs) > 0 { if includeorgs[r.GetOwner().GetLogin()] { @@ -304,13 +408,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Token: token, Origin: repo, Owner: r.GetOwner().GetLogin(), - Hoster: "github.com", + Hoster: hoster, Description: r.GetDescription(), Private: r.GetPrivate(), Issues: GetIssues(r, client, repo), NoTokenUser: true, }) - wiki := addWiki(*r, repo, token) + wiki := addWiki(*r, repo, token, hoster) if wiki.Name != "" { repos = append(repos, wiki) } @@ -323,13 +427,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Token: token, Origin: repo, Owner: r.GetOwner().GetLogin(), - Hoster: "github.com", + Hoster: hoster, Description: r.GetDescription(), Private: r.GetPrivate(), Issues: GetIssues(r, client, repo), NoTokenUser: true, }) - wiki := addWiki(*r, repo, token) + wiki := addWiki(*r, repo, token, hoster) if wiki.Name != "" { repos = append(repos, wiki) } @@ -339,28 +443,46 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if repo.Gists { gistlistoptions := &github.GistListOptions{ListOptions: github.ListOptions{PerPage: 50}} i := 1 + errorcount := 0 for { gistlistoptions.Page = i gists, _, err := client.Gists.List(context.Background(), repo.User, gistlistoptions) if err != nil { sub.Error(). Msg(err.Error()) - continue + var rateLimitErr *github.RateLimitError + var abuseRateLimitErr *github.AbuseRateLimitError + if errors.As(err, &rateLimitErr) || errors.As(err, &abuseRateLimitErr) { + sub.Warn().Msg("wait for one hour.") + time.Sleep(1 * time.Hour) + continue + } + if errorcount < 5 { + errorcount++ + time.Sleep(5 * time.Second) + continue + } + break } + errorcount = 0 if len(gists) == 0 { break } for _, gist := range gists { sub.Debug().Msg(gist.GetHTMLURL()) + gistSSHURL := fmt.Sprintf("git@gist.github.com:%s.git", gist.GetID()) + if isGHE(instURL) { + gistSSHURL = fmt.Sprintf("git@%s:gist/%s.git", hoster, gist.GetID()) + } repos = append(repos, types.Repo{ Name: fmt.Sprintf("gists%c%s", os.PathSeparator, gist.GetID()), URL: gist.GetHTMLURL(), - SSHURL: fmt.Sprintf("git@gist.github.com:%s.git", gist.GetID()), + SSHURL: gistSSHURL, Token: token, Origin: repo, Owner: gist.GetOwner().GetLogin(), - Hoster: "github.com", + Hoster: hoster, Description: gist.GetDescription(), Private: !gist.GetPublic(), NoTokenUser: true, @@ -375,30 +497,28 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { return repos, ran } -// GetOrCreate Get or create a repository -func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { - sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com") - token := destination.GetToken() - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(context.TODO(), ts) - - client := github.NewClient(tc) +// GetOrCreate returns the destination's clone URL and resolved push token, +// creating the repository if needed. +func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, string, error) { + sub = logger.CreateSubLogger("stage", "github", "url", githubInstanceURL(destination.URL)) + client, token, err := newGithubClient(context.TODO(), destination) + if err != nil { + return "", "", err + } dest := types.GithubDestination{} login := "" if destination.Organization == "" { user, _, err := client.Users.Get(context.TODO(), "") if err != nil { - return "", err + return "", "", err } dest.User = user login = *user.Login } else { organization, _, err := client.Organizations.Get(context.TODO(), destination.Organization) if err != nil { - return "", err + return "", "", err } dest.Organization = organization login = *organization.Login @@ -407,12 +527,12 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { r, _, err := client.Repositories.Get(context.TODO(), login, repo.Name) if err != nil { if !strings.Contains(err.Error(), "404 Not Found") { - return "", err + return "", "", err } if destination.Organization == "" { r, _, err = client.Repositories.Create(context.TODO(), "", &github.Repository{Name: github.String(repo.Name), Private: github.Bool(destination.Visibility.Repositories == "private"), Visibility: github.String(destination.Visibility.Repositories), Owner: dest.User}) if err != nil { - return "", err + return "", "", err } } else { if destination.Visibility.Repositories == "" { @@ -420,12 +540,12 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { } r, _, err = client.Repositories.Create(context.TODO(), *dest.Organization.Login, &github.Repository{Name: github.String(repo.Name), Private: github.Bool(destination.Visibility.Repositories == "private"), Visibility: github.String(destination.Visibility.Repositories), Organization: dest.Organization}) if err != nil { - return "", err + return "", "", err } } } - return *r.CloneURL, nil + return *r.CloneURL, token, nil } // GetIssues get issues diff --git a/github/github_test.go b/github/github_test.go new file mode 100644 index 0000000..d08cfc1 --- /dev/null +++ b/github/github_test.go @@ -0,0 +1,230 @@ +package github + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/cooperspencer/gickup/types" +) + +func TestNewGithubClientUnauthenticated(t *testing.T) { + t.Parallel() + + repo := types.GenRepo{} // no token, no app auth + + client, token, err := newGithubClient(context.Background(), repo) + if err != nil { + t.Fatalf("newGithubClient() error = %v", err) + } + + if client == nil { + t.Fatal("expected non-nil client") + } + + if token != "" { + t.Fatalf("expected empty token, got %q", token) + } +} + +func TestNewGithubClientWithToken(t *testing.T) { + t.Setenv("GITHUB_TEST_TOKEN", "my-personal-token") + + repo := types.GenRepo{Token: "GITHUB_TEST_TOKEN"} + + client, token, err := newGithubClient(context.Background(), repo) + if err != nil { + t.Fatalf("newGithubClient() error = %v", err) + } + + if client == nil { + t.Fatal("expected non-nil client") + } + + if token != "my-personal-token" { + t.Fatalf("token = %q, want %q", token, "my-personal-token") + } +} + +func TestNewGithubClientAppAuthInvalidKeyFile(t *testing.T) { + t.Parallel() + + // A file that exists but contains an invalid PEM key. + keyFile := filepath.Join(t.TempDir(), "invalid.pem") + if err := os.WriteFile(keyFile, []byte("not-a-valid-pem-key"), 0o600); err != nil { + t.Fatalf("write key file: %v", err) + } + + repo := types.GenRepo{ + AppID: 1, + AppInstallationID: 2, + AppPrivateKeyFile: keyFile, + } + + _, _, err := newGithubClient(context.Background(), repo) + if err == nil { + t.Fatal("expected error for invalid App private key file") + } +} + +func TestNewGithubClientAppAuthMissingKeyFile(t *testing.T) { + t.Parallel() + + repo := types.GenRepo{ + AppID: 1, + AppInstallationID: 2, + AppPrivateKeyFile: "/nonexistent/path/key.pem", + } + + _, _, err := newGithubClient(context.Background(), repo) + if err == nil { + t.Fatal("expected error when App private key file does not exist") + } +} + +//nolint:paralleltest // Replaces http.DefaultTransport and calls code with a package-level logger. +func TestGetOrCreateReturnsResolvedToken(t *testing.T) { + const wantToken = "installation-token-for-test" + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("rsa.GenerateKey() error = %v", err) + } + keypath := filepath.Join(t.TempDir(), "app.pem") + pemBytes := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }) + if err := os.WriteFile(keypath, pemBytes, 0o600); err != nil { + t.Fatalf("os.WriteFile() error = %v", err) + } + + tokenFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenFile, []byte(wantToken+"\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("GITHUB_DESTINATION_TEST_TOKEN", wantToken) + + for _, instance := range []struct { + name, url, apiURL, cloneURL string + }{ + {"github.com", "", "https://api.github.com", "https://github.com/example/repo.git"}, + {"enterprise-server", "https://github.example.com/", "https://github.example.com/api/v3", "https://github.example.com/example/repo.git"}, + } { + for _, tc := range []struct { + name, auth, failAt string + create bool + }{ + {name: "app-existing", auth: "app"}, + {name: "app-create", auth: "app", create: true}, + {name: "pat-existing", auth: "pat"}, + {name: "pat-create", auth: "pat", create: true}, + {name: "environment-token", auth: "environment"}, + {name: "token-file", auth: "file"}, + {name: "token-error", auth: "app", failAt: "/app/installations/4242/access_tokens"}, + {name: "organization-error", auth: "app", failAt: "/orgs/example"}, + {name: "repository-error", auth: "app", failAt: "/repos/example/repo"}, + {name: "creation-error", auth: "app", create: true, failAt: "/orgs/example/repos"}, + } { + t.Run(instance.name+"/"+tc.name, func(t *testing.T) { + destination := types.GenRepo{URL: instance.url, Organization: "example"} + switch tc.auth { + case "app": + destination.AppID = 99 + destination.AppInstallationID = 4242 + destination.AppPrivateKeyFile = keypath + destination.Token = "unused-configured-token" + case "pat": + destination.Token = wantToken + case "environment": + destination.Token = "GITHUB_DESTINATION_TEST_TOKEN" + case "file": + destination.TokenFile = tokenFile + } + + created, failed := false, false + // These fixtures rely on clients using http.DefaultTransport to intercept github.com requests. + oldTransport := http.DefaultTransport + t.Cleanup(func() { http.DefaultTransport = oldTransport }) + http.DefaultTransport = roundTripperFunc(func(r *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + w.Header().Set("Content-Type", "application/json") + if !strings.HasPrefix(r.URL.String(), instance.apiURL+"/") { + t.Errorf("unexpected API URL: %s", r.URL) + w.WriteHeader(http.StatusNotFound) + return w.Result(), nil + } + endpoint := strings.TrimPrefix(r.URL.String(), instance.apiURL) + if endpoint == "/app/installations/4242/access_tokens" { + if r.Method != http.MethodPost || !strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ") { + t.Error("installation token request must use POST and App JWT authentication") + } + } else if !strings.HasSuffix(r.Header.Get("Authorization"), " "+wantToken) { + t.Error("API request did not use the resolved token") + } + if endpoint == tc.failAt { + failed = true + w.WriteHeader(http.StatusForbidden) + return w.Result(), nil + } + switch r.Method + " " + endpoint { + case "POST /app/installations/4242/access_tokens": + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]string{ + "token": wantToken, "expires_at": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), + }) + case "GET /orgs/example": + _, _ = w.WriteString(`{"login":"example"}`) + case "GET /repos/example/repo": + if tc.create { + w.WriteHeader(http.StatusNotFound) + _, _ = w.WriteString(`{"message":"Not Found"}`) + } else { + _ = json.NewEncoder(w).Encode(map[string]string{"clone_url": instance.cloneURL}) + } + case "POST /orgs/example/repos": + created = true + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]string{"clone_url": instance.cloneURL}) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + return w.Result(), nil + }) + + cloneURL, token, err := GetOrCreate(destination, types.Repo{Name: "repo"}) + if tc.failAt != "" { + if !failed || err == nil || cloneURL != "" || token != "" { + t.Fatalf("API error returned (%q, %q, %v), want empty URL and token with an error", cloneURL, token, err) + } + return + } + if err != nil { + t.Fatal(err) + } + if cloneURL != instance.cloneURL || token != wantToken { + t.Fatalf("GetOrCreate() = (%q, %q), want (%q, %q)", cloneURL, token, instance.cloneURL, wantToken) + } + if created != tc.create { + t.Fatalf("repository created = %v, want %v", created, tc.create) + } + }) + } + } +} + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index b35518b..56db880 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -16,6 +16,20 @@ import ( var sub zerolog.Logger +func getRepoVisibility(visibility string, private bool) gitlab.VisibilityValue { + switch strings.ToLower(strings.TrimSpace(visibility)) { + case "private": + return gitlab.PrivateVisibility + case "public": + return gitlab.PublicVisibility + default: + if private { + return gitlab.PrivateVisibility + } + return gitlab.PublicVisibility + } +} + // Backup TODO. func Backup(r types.Repo, d types.GenRepo, dry bool) bool { var gitlabclient *gitlab.Client @@ -69,13 +83,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { splittedurl[0], r.Owner, r.Token, splittedurl[1]) } - var visibility gitlab.VisibilityValue - - if r.Private { - visibility = gitlab.PrivateVisibility - } else { - visibility = gitlab.PublicVisibility - } + visibility := getRepoVisibility(d.Visibility.Repositories, r.Private) opts := &gitlab.CreateProjectOptions{ Mirror: &True, @@ -209,7 +217,16 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Msg(err.Error()) continue } - subgroups, _, err := client.Groups.ListSubGroups(group.ID, &gitlab.ListSubGroupsOptions{}) + + subgroups, err := gitlab.ScanAndCollect(func(p gitlab.PaginationOptionFunc) ([]*gitlab.Group, *gitlab.Response, error) { + return client.Groups.ListDescendantGroups(group.ID, &gitlab.ListDescendantGroupsOptions{}, p) + }) + if err != nil { + sub.Error(). + Msg(err.Error()) + continue + } + for _, sub := range subgroups { includeorgs[sub.FullPath] = true } @@ -222,7 +239,16 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Msg(err.Error()) continue } - subgroups, _, err := client.Groups.ListSubGroups(group.ID, &gitlab.ListSubGroupsOptions{}) + + subgroups, err := gitlab.ScanAndCollect(func(p gitlab.PaginationOptionFunc) ([]*gitlab.Group, *gitlab.Response, error) { + return client.Groups.ListDescendantGroups(group.ID, &gitlab.ListDescendantGroupsOptions{}, p) + }) + if err != nil { + sub.Error(). + Msg(err.Error()) + continue + } + for _, sub := range subgroups { excludeorgs[sub.FullPath] = true } @@ -269,6 +295,12 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if time.Since(*r.LastActivityAt) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { continue } + if exclude[r.Name] { + continue + } + if excludeorgs[r.Namespace.FullPath] { + continue + } if include[r.Name] { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { repos = append(repos, types.Repo{ @@ -305,9 +337,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } if len(include) == 0 { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { repos = append(repos, types.Repo{ @@ -436,6 +465,12 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } + if exclude[r.Name] { + continue + } + if excludeorgs[r.Namespace.FullPath] { + continue + } if include[r.Name] { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { repos = append(repos, types.Repo{ @@ -472,13 +507,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if excludeorgs[r.Namespace.FullPath] { - continue - } - if len(include) == 0 { if len(includeorgs) == 0 || includeorgs[r.Namespace.FullPath] { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { @@ -562,7 +590,7 @@ func GetIssues(repo *gitlab.Project, client *gitlab.Client, conf types.GenRepo) } else { if len(i) > 0 { for _, issue := range i { - issues[strconv.FormatInt(issue.IID, 64)] = issue + issues[strconv.FormatInt(issue.IID, 10)] = issue } } else { break @@ -576,11 +604,7 @@ func GetIssues(repo *gitlab.Project, client *gitlab.Client, conf types.GenRepo) // GetOrCreate Get or create a repository func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { - visibility := gitlab.PublicVisibility - - if repo.Private { - visibility = gitlab.PrivateVisibility - } + visibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private) sub = logger.CreateSubLogger("stage", "gitlab", "url", destination.URL) diff --git a/gitlab/gitlab_test.go b/gitlab/gitlab_test.go new file mode 100644 index 0000000..22ac98e --- /dev/null +++ b/gitlab/gitlab_test.go @@ -0,0 +1,35 @@ +package gitlab + +import ( + "testing" + + gitlab "gitlab.com/gitlab-org/api/client-go" +) + +func TestGetRepoVisibility(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + configured string + sourcePrivate bool + want gitlab.VisibilityValue + }{ + {name: "private overrides public source", configured: "private", want: gitlab.PrivateVisibility}, + {name: "public overrides private source", configured: "public", sourcePrivate: true, want: gitlab.PublicVisibility}, + {name: "empty follows private source", sourcePrivate: true, want: gitlab.PrivateVisibility}, + {name: "empty follows public source", want: gitlab.PublicVisibility}, + {name: "source follows private source", configured: "source", sourcePrivate: true, want: gitlab.PrivateVisibility}, + {name: "source follows public source", configured: "source", want: gitlab.PublicVisibility}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + if got := getRepoVisibility(test.configured, test.sourcePrivate); got != test.want { + t.Fatalf("getRepoVisibility(%q, %t) = %q, want %q", test.configured, test.sourcePrivate, got, test.want) + } + }) + } +} diff --git a/go.mod b/go.mod index f65a4d6..7f7658d 100644 --- a/go.mod +++ b/go.mod @@ -3,28 +3,30 @@ module github.com/cooperspencer/gickup go 1.26 require ( - code.gitea.io/sdk/gitea v0.24.1 + code.gitea.io/sdk/gitea v0.25.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 github.com/alecthomas/kong v1.15.0 + github.com/bradleyfalzon/ghinstallation/v2 v2.18.0 github.com/cooperspencer/onedev v0.0.0-20240801121041-919f6d3f7ea6 - github.com/go-git/go-git/v5 v5.18.0 + github.com/go-git/go-git/v5 v5.19.2 github.com/goccy/go-yaml v1.19.2 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/google/go-cmp v0.7.0 github.com/google/go-github/v74 v74.0.0 - github.com/gookit/color v1.6.0 + github.com/gookit/color v1.6.1 github.com/hasura/go-graphql-client v0.16.0 - github.com/ktrysmt/go-bitbucket v0.9.96 + github.com/ktrysmt/go-bitbucket v0.10.0 github.com/melbahja/goph v1.5.0 github.com/minio/minio-go/v7 v7.0.100 github.com/prometheus/client_golang v1.23.2 github.com/robfig/cron/v3 v3.0.1 - github.com/rs/zerolog v1.35.0 + github.com/rs/zerolog v1.35.1 github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed + github.com/studio-b12/gowebdav v0.13.0 gitlab.com/gitlab-org/api/client-go v1.46.0 - golang.org/x/crypto v0.50.0 + golang.org/x/crypto v0.53.0 golang.org/x/oauth2 v0.36.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -33,7 +35,7 @@ require ( dario.cat/mergo v1.0.2 // indirect github.com/42wim/httpsig v1.2.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -46,10 +48,12 @@ require ( github.com/emirpasic/gods v1.18.1 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.8.0 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-ini/ini v1.67.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/go-github/v84 v84.0.0 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -57,24 +61,24 @@ require ( github.com/hashicorp/go-version v1.9.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.6.0 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.21 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/philhofer/fwd v1.2.0 // indirect - github.com/pjbgf/sha1cd v0.5.0 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/sftp v1.13.10 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/common v0.68.0 // indirect github.com/prometheus/procfs v0.20.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sergi/go-diff v1.4.0 // indirect @@ -83,11 +87,10 @@ require ( github.com/tinylib/msgp v1.6.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.39.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 863db69..b6e1f86 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -code.gitea.io/sdk/gitea v0.24.1 h1:hpaqcdGcBmfMpV7JSbBJVwE99qo+WqGreJYKrDKEyW8= -code.gitea.io/sdk/gitea v0.24.1/go.mod h1:5/77BL3sHneCMEiZaMT9lfTvnnibsYxyO48mceCF3qA= +code.gitea.io/sdk/gitea v0.25.1 h1:yywxWwoV+SdjHtbC6unBiXojWdZOtoHuGhEazEXeWuE= +code.gitea.io/sdk/gitea v0.25.1/go.mod h1:uDFWYBU8dgZsgOHwe6C/6olxvf8FHguNB3wW1i83fgg= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/42wim/httpsig v1.2.4 h1:mI5bH0nm4xn7K18fo1K3okNDRq8CCJ0KbBYWyA6r8lU= @@ -14,12 +14,12 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6Xu github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 h1:BM85pSYlVYQHdq00nxyPoOkyLF5NArJG3bOsrmbwr4k= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0/go.mod h1:QYjP2cB7ZYtS/8jAbE0VSBZde/tjExqGjp+8JY6/+ts= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1 h1:edShSHV3DV90+kt+CMaEXEzR9QF7wFrPJxVGz2blMIU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -37,6 +37,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bradleyfalzon/ghinstallation/v2 v2.18.0 h1:WPqnN6NS9XvYlOgZQAIseN7Z1uAiE+UxgDKlW7FvFuU= +github.com/bradleyfalzon/ghinstallation/v2 v2.18.0/go.mod h1:gpoSwwWc4biE49F7n+roCcpkEkZ1Qr9soZ2ESvMiouU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= @@ -66,18 +68,20 @@ github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0= -github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.18.0 h1:O831KI+0PR51hM2kep6T8k+w0/LIAD490gvqMCvL5hM= -github.com/go-git/go-git/v5 v5.18.0/go.mod h1:pW/VmeqkanRFqR6AljLcs7EA7FbZaN5MQqO7oZADXpo= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 h1:UjoPNDAQ5JPCjlxoJd6K8ALZqSDDhk2ymieAZOVaDg0= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85/go.mod h1:fR6z1Ie6rtF7kl/vBYMfgD5/G5B1blui7z426/sj2DU= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= @@ -87,14 +91,16 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v74 v74.0.0 h1:yZcddTUn8DPbj11GxnMrNiAnXH14gNs559AsUpNpPgM= github.com/google/go-github/v74 v74.0.0/go.mod h1:ubn/YdyftV80VPSI26nSJvaEsTOnsjrxG3o9kJhcyak= +github.com/google/go-github/v84 v84.0.0 h1:I/0Xn5IuChMe8TdmI2bbim5nyhaRFJ7DEdzmD2w+yVA= +github.com/google/go-github/v84 v84.0.0/go.mod h1:WwYL1z1ajRdlaPszjVu/47x1L0PXukJBn73xsiYrRRQ= github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0= github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E= -github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA= -github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= +github.com/gookit/color v1.6.1 h1:KoTnDxJPRgrL0SoX0f8rCFg2zI0t4E3GZZBMo2nN8LU= +github.com/gookit/color v1.6.1/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= github.com/graph-gophers/graphql-go v1.9.0 h1:yu0ucKHLc5qGpRwLYKIWtr9bOoxovkWasuBrPQwlHls= github.com/graph-gophers/graphql-go v1.9.0/go.mod h1:23olKZ7duEvHlF/2ELEoSZaY1aNPfShjP782SOoNTyM= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -115,8 +121,8 @@ github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRg github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -131,14 +137,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/ktrysmt/go-bitbucket v0.9.96 h1:KI+ePerKCIJHkZNFV/fJkk/8yfWTQDgr139UOIp/RJY= -github.com/ktrysmt/go-bitbucket v0.9.96/go.mod h1:/1H7KMC7FsdkRoNMq21an6dyFpA+irNw5tyYpHzRC2Q= +github.com/ktrysmt/go-bitbucket v0.10.0 h1:miXSfqFb+j0pPBB9h8kw9f/oeq045WOPI4hplr996+k= +github.com/ktrysmt/go-bitbucket v0.10.0/go.mod h1:IUB8I+gC3UO00NjNTMS7STjsZYq+EAhPuHgynzRWxqY= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= -github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/melbahja/goph v1.5.0 h1:RQUBpLvfg3i7fjfG8rTcSWyMjVRfdhwrrfQhjYee4dQ= github.com/melbahja/goph v1.5.0/go.mod h1:dDwo+44cmvfDLdiVpc6fJxexf5BA5yEDUeE5YgtuDO4= github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= @@ -155,8 +161,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pjbgf/sha1cd v0.5.0 h1:a+UkboSi1znleCDUNT3M5YxjOnN1fz2FhN48FlwCxs0= -github.com/pjbgf/sha1cd v0.5.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -169,8 +175,8 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= -github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/common v0.68.0 h1:8rQJvQmYltsR2L7h8Zw0Iyj8WYNNmpwikoQTZXwfVeA= +github.com/prometheus/common v0.68.0/go.mod h1:4soH+U8yJSROk7OJ//hmTiWKsxapv6zRGgTt3keN8gQ= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -179,8 +185,8 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/rs/zerolog v1.35.0 h1:VD0ykx7HMiMJytqINBsKcbLS+BJ4WYjz+05us+LRTdI= -github.com/rs/zerolog v1.35.0/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed h1:KT7hI8vYXgU0s2qaMkrfq9tCA1w/iEPgfredVP+4Tzw= @@ -195,6 +201,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/studio-b12/gowebdav v0.13.0 h1:OcwSg6IQHOFNdYHn3bPOHwSE8looG8N56Y5xTT1asqQ= +github.com/studio-b12/gowebdav v0.13.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= @@ -213,15 +221,15 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= -golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 h1:SbTAbRFnd5kjQXbczszQ0hdk3ctwYf3qBNH9jIsGclE= -golang.org/x/exp v0.0.0-20250813145105-42675adae3e6/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -233,16 +241,16 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/gogs/gogs.go b/gogs/gogs.go index 415f2c8..360f7e4 100644 --- a/gogs/gogs.go +++ b/gogs/gogs.go @@ -182,6 +182,11 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { continue } + + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -214,10 +219,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, @@ -309,6 +310,10 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -341,10 +346,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(repo.Include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, diff --git a/httpclient/useragent.go b/httpclient/useragent.go new file mode 100644 index 0000000..6ebcd9b --- /dev/null +++ b/httpclient/useragent.go @@ -0,0 +1,37 @@ +package httpclient + +import ( + "net/http" + "sync" + + gitclient "github.com/go-git/go-git/v5/plumbing/transport/client" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" +) + +// Custom UserAgent for tracking gickup calls in the network +// Some Services may only serve git-over-HTTP endpoints to clients whose User-Agent starts with "git/" +const gickupUserAgent = "git/gickup" + +type userAgentTransport struct { + base http.RoundTripper +} + +func (t userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) { + clone := req.Clone(req.Context()) + clone.Header.Set("User-Agent", gickupUserAgent) + return t.base.RoundTrip(clone) +} + +// Run once sync used to gate init of the UserAgent for go-git +var installUserAgentOnce sync.Once + +// ApplyGickupUserAgent installs go-git HTTP(S) clients that send a git-compatible User-Agent. +// "git/" User-Agent is what the real git client sends, so it is accepted by every other hoster. +func ApplyGickupUserAgent() { + installUserAgentOnce.Do(func() { + hc := &http.Client{Transport: userAgentTransport{base: http.DefaultTransport}} + c := githttp.NewClient(hc) + gitclient.InstallProtocol("http", c) + gitclient.InstallProtocol("https", c) + }) +} diff --git a/httpclient/useragent_test.go b/httpclient/useragent_test.go new file mode 100644 index 0000000..22af991 --- /dev/null +++ b/httpclient/useragent_test.go @@ -0,0 +1,29 @@ +package httpclient + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestUserAgentTransportRewrites(t *testing.T) { + t.Parallel() + var got string + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + got = r.Header.Get("User-Agent") + })) + defer srv.Close() + + rt := userAgentTransport{base: http.DefaultTransport} + req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, srv.URL, nil) + req.Header.Set("User-Agent", "go-git/5.x") + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("roundtrip: %v", err) + } + resp.Body.Close() + + if got != gickupUserAgent { + t.Fatalf("expected User-Agent %q, got %q", gickupUserAgent, got) + } +} diff --git a/local/local.go b/local/local.go index 00f1d7d..b922056 100644 --- a/local/local.go +++ b/local/local.go @@ -34,6 +34,33 @@ var ( sub zerolog.Logger ) +// tokenAuth returns the HTTP basic auth for a repo that authenticates via token. +func tokenAuth(repo types.Repo) *http.BasicAuth { + if repo.Token == "" { + return nil + } + if repo.NoTokenUser { + return &http.BasicAuth{ + Username: "x-access-token", + Password: repo.Token, + } + } + return &http.BasicAuth{ + Username: repo.Origin.User, + Password: repo.Token, + } +} + +func toGitCmdAuth(auth transport.AuthMethod) *gitcmd.Auth { + if basicAuth, ok := auth.(*http.BasicAuth); ok && basicAuth != nil { + return &gitcmd.Auth{ + Username: basicAuth.Username, + Password: basicAuth.Password, + } + } + return nil +} + // Locally TODO. func Locally(repo types.Repo, l types.Local, dry bool) bool { sub = logger.CreateSubLogger("stage", "locally", "path", l.Path) @@ -97,16 +124,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { return false } case repo.Token != "": - if repo.NoTokenUser { - auth = &http.BasicAuth{ - Username: repo.Token, - } - } else { - auth = &http.BasicAuth{ - Username: repo.Origin.User, - Password: repo.Token, - } - } + auth = tokenAuth(repo) case repo.Origin.Username != "" && repo.Origin.Password != "": auth = &http.BasicAuth{ Username: repo.Origin.Username, @@ -179,7 +197,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { sub.Info(). Msgf("opening %s locally", types.Green(repo.Name)) - err := updateRepository(repo.Name, auth, dry, l) + err := updateRepository(repo, auth, dry, l) if err != nil { switch { case errors.Is(err, git.NoErrAlreadyUpToDate): @@ -260,84 +278,126 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { } if l.Keep > 0 { - parentdir := path.Dir(filepath.Join(l.Path, repo.Name)) - files, err := os.ReadDir(parentdir) - if err != nil { + if err := pruneOldBackups(filepath.Dir(filepath.Join(l.Path, repo.Name)), repo.Name, l); err != nil { sub.Warn(). Str("repo", repo.Name).Msg(err.Error()) return false } + } - keep := []string{} - for _, file := range files { - fname := file.Name() - if l.Zip { - fname = strings.TrimSuffix(file.Name(), ".zip") - } + x = 5 + } - fname = strings.TrimSuffix(fname, ".issues") + return true +} - _, err := strconv.ParseInt(fname, 10, 64) - if err != nil { - sub.Warn(). - Str("repo", repo.Name). - Msgf("couldn't parse timestamp! %s", types.Red(file.Name())) - } - if l.Zip && !strings.HasSuffix(file.Name(), ".zip") { - continue - } - keep = append(keep, file.Name()) - } +// pruneOldBackups enforces l.Keep by removing the oldest timestamped backups +func pruneOldBackups(parentdir string, repoName string, l types.Local) error { + files, err := os.ReadDir(parentdir) + if err != nil { + return err + } - sort.Sort(sort.Reverse(sort.StringSlice(keep))) + // A backup generation is the "" directory plus "." + // (like ".issues") + // They are grouped by their timestamp so that together they only count as one backup against l.Keep. + generations := map[int64][]string{} + for _, file := range files { + stamp, _, _ := strings.Cut(file.Name(), ".") - if len(keep) > l.Keep { - toremove := keep[l.Keep:] - for _, file := range toremove { - sub.Info(). - Msgf("removing %s", types.Red(path.Join(parentdir, file))) - err := os.RemoveAll(path.Join(parentdir, file)) - if err != nil { - sub.Warn(). - Str("repo", repo.Name).Msg(err.Error()) - } - } - } + ts, err := strconv.ParseInt(stamp, 10, 64) + if err != nil { + sub.Warn(). + Str("repo", repoName). + Msgf("couldn't parse timestamp! %s", types.Red(file.Name())) + continue } + generations[ts] = append(generations[ts], file.Name()) + } - x = 5 + if len(generations) <= l.Keep { + return nil } - return true + stamps := make([]int64, 0, len(generations)) + for ts := range generations { + stamps = append(stamps, ts) + } + sort.Slice(stamps, func(i, j int) bool { return stamps[i] > stamps[j] }) + + for _, ts := range stamps[l.Keep:] { + for _, file := range generations[ts] { + sub.Info(). + Msgf("removing %s", types.Red(filepath.Join(parentdir, file))) + err := os.RemoveAll(filepath.Join(parentdir, file)) + if err != nil { + sub.Warn(). + Str("repo", repoName).Msg(err.Error()) + } + } + } + + return nil } -func updateRepository(reponame string, auth transport.AuthMethod, dry bool, l types.Local) error { - r, err := git.PlainOpen(filepath.Join(l.Path, reponame)) +func sanitizeRemote(r *git.Repository, cleanURL string) error { + if cleanURL == "" { + return nil + } + cfg, err := r.Config() + if err != nil { + return err + } + rem, ok := cfg.Remotes["origin"] + if !ok || rem == nil { + return nil + } + changed := false + for i, u := range rem.URLs { + if (strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://")) && strings.Contains(u, "@") { + rem.URLs[i] = cleanURL + changed = true + } + } + if changed { + return r.SetConfig(cfg) + } + return nil +} + +func updateRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l types.Local) error { + r, err := git.PlainOpen(filepath.Join(l.Path, repo.Name)) if err != nil { return err } if !dry { if l.LFS { - _, err = os.Stat(filepath.Join(l.Path, reponame)) + _, err = os.Stat(filepath.Join(l.Path, repo.Name)) if err != nil { return err } + if !repo.Origin.SSH && repo.URL != "" { + if err := sanitizeRemote(r, repo.URL); err != nil { + sub.Warn().Str("repo", repo.Name).Msgf("could not sanitize remote: %s", err) + } + } + sub.Info(). - Msgf("pulling %s", types.Green(reponame)) + Msgf("pulling %s", types.Green(repo.Name)) - err = gitc.Pull(l.Bare, l.Mirror, filepath.Join(l.Path, reponame)) + err = gitc.Pull(l.Bare, l.Mirror, filepath.Join(l.Path, repo.Name), toGitCmdAuth(auth)) if err != nil { return err } if l.Bare || l.Mirror { sub.Info(). - Msgf("fetching lfs files for %s", types.Green(reponame)) + Msgf("fetching lfs files for %s", types.Green(repo.Name)) - err = gitc.LFSFetch(filepath.Join(l.Path, reponame)) + err = gitc.LFSFetch(filepath.Join(l.Path, repo.Name), toGitCmdAuth(auth)) if err != nil { return err } @@ -349,7 +409,7 @@ func updateRepository(reponame string, auth transport.AuthMethod, dry bool, l ty return err } sub.Info(). - Msgf("pulling %s", types.Green(reponame)) + Msgf("pulling %s", types.Green(repo.Name)) if !l.Bare && !l.Mirror { w, worktreeErr := r.Worktree() if worktreeErr != nil && !errors.Is(worktreeErr, git.NoErrAlreadyUpToDate) { @@ -409,25 +469,7 @@ func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l typ } if l.LFS { - if repo.Token != "" { - if strings.HasPrefix(url, "http://") { - url = strings.ReplaceAll(url, "http://", fmt.Sprintf("http://xyz:%s@", repo.Token)) - } - - if strings.HasPrefix(url, "https://") { - url = strings.ReplaceAll(url, "https://", fmt.Sprintf("https://xyz:%s@", repo.Token)) - } - } else if repo.Origin.Username != "" && repo.Origin.Password != "" { - if strings.HasPrefix(url, "http://") { - url = strings.ReplaceAll(url, "http://", fmt.Sprintf("http://%s:%s@", repo.Origin.Username, repo.Origin.Password)) - } - - if strings.HasPrefix(url, "https://") { - url = strings.ReplaceAll(url, "https://", fmt.Sprintf("https://%s:%s@", repo.Origin.Username, repo.Origin.Password)) - } - } - - err = gitc.Clone(url, filepath.Join(l.Path, repo.Name), l.Bare, l.Mirror) + err = gitc.Clone(url, filepath.Join(l.Path, repo.Name), l.Bare, l.Mirror, toGitCmdAuth(auth)) if err != nil { return err } @@ -436,7 +478,7 @@ func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l typ sub.Info(). Msgf("fetching lfs files for %s", types.Green(repo.Name)) - err = gitc.LFSFetch(filepath.Join(l.Path, repo.Name)) + err = gitc.LFSFetch(filepath.Join(l.Path, repo.Name), toGitCmdAuth(auth)) if err != nil { return err } @@ -518,15 +560,11 @@ func TempCloneBare(repo types.Repo, tempdir string) (*git.Repository, error) { func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repository, error) { var auth transport.AuthMethod if repo.Token != "" { - if repo.NoTokenUser { - auth = &http.BasicAuth{ - Username: repo.Token, - } - } else { - auth = &http.BasicAuth{ - Username: repo.Origin.User, - Password: repo.Token, - } + auth = tokenAuth(repo) + } else if repo.Origin.Username != "" && repo.Origin.Password != "" { + auth = &http.BasicAuth{ + Username: repo.Origin.Username, + Password: repo.Origin.Password, } } if repo.Origin.LFS { @@ -536,15 +574,7 @@ func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repositor } gitc = g - if strings.HasPrefix(repo.URL, "http://") { - repo.URL = strings.ReplaceAll(repo.URL, "http://", fmt.Sprintf("http://xyz:%s@", repo.Token)) - } - - if strings.HasPrefix(repo.URL, "https://") { - repo.URL = strings.ReplaceAll(repo.URL, "https://", fmt.Sprintf("https://xyz:%s@", repo.Token)) - } - - err = gitc.Clone(repo.URL, tempdir, isBare, false) + err = gitc.Clone(repo.URL, tempdir, isBare, false, toGitCmdAuth(auth)) if err != nil { return nil, err } @@ -558,7 +588,25 @@ func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repositor Auth: auth, Force: true, }) - if errors.Is(err, git.NoErrAlreadyUpToDate) { + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return r, err + } + + // A regular clone only downloads the LFS objects needed for the + // checked-out branch. Without fetching the rest, objects referenced + // by other branches/tags or older commits are missing locally, + // which makes the later `git push --all` fail with + // "missing or corrupt local objects". + sub.Info(). + Msgf("fetching lfs files for %s", types.Green(repo.Name)) + + if lfsErr := gitc.LFSFetch(tempdir, toGitCmdAuth(auth)); lfsErr != nil { + return r, lfsErr + } + + // Bare backups already contain all refs and LFS objects. + // Checkout and pull require a working tree and must not run for these clones. + if isBare { return r, nil } @@ -590,7 +638,7 @@ func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repositor return nil, err } - err = gitc.MirrorPull(tempdir) + err = gitc.MirrorPull(tempdir, toGitCmdAuth(auth)) if err != nil { return nil, err } @@ -617,9 +665,9 @@ func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repositor return r, err } -func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url string, lfs bool) error { +// CreateRemotePush pushes to url using the caller's resolved token or SSH configuration. +func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url, token string, lfs bool) error { sub = logger.CreateSubLogger("stage", "tempclone", "url", url) - token := destination.GetToken() var auth transport.AuthMethod if destination.SSH { if destination.SSHKey == "" { @@ -681,20 +729,12 @@ func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url strin return err } } else { - if strings.HasPrefix(url, "http://") { - url = strings.ReplaceAll(url, "http://", fmt.Sprintf("http://xyz:%s@", token)) - } - - if strings.HasPrefix(url, "https://") { - url = strings.ReplaceAll(url, "https://", fmt.Sprintf("https://xyz:%s@", token)) - } - err = gitc.NewRemote(remote, url, worktree.Filesystem.Root()) if err != nil { return err } - err = gitc.Push(worktree.Filesystem.Root(), remote) + err = gitc.Push(worktree.Filesystem.Root(), remote, toGitCmdAuth(auth)) if err != nil { return err } @@ -714,13 +754,45 @@ func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url strin err = repo.Push(&pushoptions) if err == nil || errors.Is(err, git.NoErrAlreadyUpToDate) { - pushoptions = git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}} + refspecs, err := getPushRefSpecs(repo, headref.Name()) + if err != nil { + return err + } - return repo.Push(&pushoptions) + var pushErrors []error + // Keep valid refs moving when a destination hook rejects one branch or tag. + for _, refspec := range refspecs { + pushoptions.RefSpecs = []config.RefSpec{refspec} + err = repo.Push(&pushoptions) + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + pushErrors = append(pushErrors, err) + } + } + + return errors.Join(pushErrors...) } return err } +func getPushRefSpecs(repo *git.Repository, skip plumbing.ReferenceName) ([]config.RefSpec, error) { + refs, err := repo.References() + if err != nil { + return nil, err + } + + refspecs := []config.RefSpec{} + err = refs.ForEach(func(ref *plumbing.Reference) error { + name := ref.Name() + if name == skip || (!name.IsBranch() && !name.IsTag()) { + return nil + } + + refspecs = append(refspecs, config.RefSpec(fmt.Sprintf("%s:%s", name, name))) + return nil + }) + return refspecs, err +} + func RandomString(length int) string { charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) diff --git a/local/local_push_test.go b/local/local_push_test.go new file mode 100644 index 0000000..7c4fa5f --- /dev/null +++ b/local/local_push_test.go @@ -0,0 +1,90 @@ +package local + +import ( + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/cooperspencer/gickup/types" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" +) + +//nolint:paralleltest // CreateRemotePush assigns the package-level logger. +func TestCreateRemotePushUsesProvidedToken(t *testing.T) { + const want = "resolved-token-for-test" + + dir := t.TempDir() + repo, err := git.PlainInit(dir, false) + if err != nil { + t.Fatalf("git.PlainInit() error = %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte("x\n"), 0o644); err != nil { + t.Fatalf("os.WriteFile() error = %v", err) + } + wt, err := repo.Worktree() + if err != nil { + t.Fatalf("Worktree() error = %v", err) + } + if _, err := wt.Add("README.md"); err != nil { + t.Fatalf("Add() error = %v", err) + } + if _, err := wt.Commit("seed", &git.CommitOptions{ + Author: &object.Signature{Name: "t", Email: "t@example.com", When: time.Now()}, + }); err != nil { + t.Fatalf("Commit() error = %v", err) + } + + var passwordsMu sync.Mutex + var passwords []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, password, _ := r.BasicAuth() + passwordsMu.Lock() + passwords = append(passwords, password) + passwordsMu.Unlock() + w.Header().Set("WWW-Authenticate", `Basic realm="git"`) + w.WriteHeader(http.StatusUnauthorized) + })) + defer server.Close() + + for _, configuredToken := range []string{"", "unused-configured-token"} { + destination := types.GenRepo{Token: configuredToken} + err := CreateRemotePush(repo, destination, server.URL+"/org/repo.git", want, false) + if !errors.Is(err, transport.ErrAuthenticationRequired) { + t.Fatalf("push error = %v, want authentication required", err) + } + passwordsMu.Lock() + gotPasswords := passwords + passwords = nil + passwordsMu.Unlock() + if len(gotPasswords) == 0 { + t.Fatal("push did not reach the server") + } + for _, got := range gotPasswords { + if got != want { + t.Fatalf("push password = %q, want %q", got, want) + } + } + } + + reopened, err := git.PlainOpen(dir) + if err != nil { + t.Fatal(err) + } + remotes, err := reopened.Remotes() + if err != nil { + t.Fatal(err) + } + for _, remote := range remotes { + urls := remote.Config().URLs + if len(urls) != 1 || urls[0] != server.URL+"/org/repo.git" { + t.Fatalf("persisted remote URLs = %v, want original credential-free URL", urls) + } + } +} diff --git a/local/local_test.go b/local/local_test.go index f1a1be9..79569b5 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -1,10 +1,117 @@ package local import ( + "os" + "path/filepath" + "reflect" "strings" "testing" + + "github.com/cooperspencer/gickup/types" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" + "github.com/go-git/go-git/v5/storage/memory" ) +func runPrune(t *testing.T, backups []string, keep int) []string { + t.Helper() + + parentdir := t.TempDir() + for _, d := range backups { + if err := os.Mkdir(filepath.Join(parentdir, d), 0o777); err != nil { + t.Fatal(err) + } + } + + if err := pruneOldBackups(parentdir, "test/repo", types.Local{Keep: keep}); err != nil { + t.Fatal(err) + } + + files, err := os.ReadDir(parentdir) + if err != nil { + t.Fatal(err) + } + remaining := []string{} + for _, f := range files { + remaining = append(remaining, f.Name()) + } + + return remaining +} + +func TestPruneKeepsNewestBackup(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, []string{"1785061240", "1785115104"}, 1) + + want := []string{"1785115104"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + +func TestPruneKeepsIssuesDirWithItsBackup(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, + []string{"1785061241", "1785061241.issues", "1785115111", "1785115111.issues"}, 1) + + want := []string{"1785115111", "1785115111.issues"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + +func TestPruneKeepsOtherDirectories(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, + []string{"1785061241", "1785061241.issues", "dummy", "dummy.issue", "1785061241x.dummy", "1785115111", "1785115111.issues"}, 1) + + want := []string{"1785061241x.dummy", "1785115111", "1785115111.issues", "dummy", "dummy.issue"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + +func TestPruneDeletesOtherTimestampedDirectories(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, + []string{"1785061241", "1785061241.issues", "1785061241.dummy", "1785115111", "1785115111.issues"}, 1) + + want := []string{"1785115111", "1785115111.issues"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + +func TestPruneKeepsMultipleGenerations(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, + []string{"1785000000", "1785000000.issues", "1785061241", "1785061241.issues", "1785115111", "1785115111.issues"}, 2) + + want := []string{"1785061241", "1785061241.issues", "1785115111", "1785115111.issues"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + +func TestPruneKeepsEverythingBelowKeepCount(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, []string{"1785115111", "1785115111.issues"}, 2) + + want := []string{"1785115111", "1785115111.issues"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + func TestRandomStringLengthAndCharset(t *testing.T) { t.Parallel() @@ -29,3 +136,290 @@ func TestRandomStringZeroLength(t *testing.T) { t.Fatalf("RandomString(0) = %q, want empty string", got) } } + +func TestTokenAuth_NoToken(t *testing.T) { + t.Parallel() + + repo := types.Repo{Token: ""} + if got := tokenAuth(repo); got != nil { + t.Fatalf("tokenAuth with empty token = %v, want nil", got) + } +} + +func TestTokenAuth_NoTokenUser(t *testing.T) { + t.Parallel() + + repo := types.Repo{ + Token: "mytoken", + NoTokenUser: true, + } + got := tokenAuth(repo) + if got == nil { + t.Fatal("tokenAuth returned nil, want *BasicAuth") + } + if got.Username != "x-access-token" { + t.Errorf("Username = %q, want %q", got.Username, "x-access-token") + } + if got.Password != "mytoken" { + t.Errorf("Password = %q, want %q", got.Password, "mytoken") + } +} + +func TestTokenAuth_WithTokenUser(t *testing.T) { + t.Parallel() + + repo := types.Repo{ + Token: "mytoken", + NoTokenUser: false, + Origin: types.GenRepo{ + User: "octocat", + }, + } + got := tokenAuth(repo) + if got == nil { + t.Fatal("tokenAuth returned nil, want *BasicAuth") + } + if got.Username != "octocat" { + t.Errorf("Username = %q, want %q", got.Username, "octocat") + } + if got.Password != "mytoken" { + t.Errorf("Password = %q, want %q", got.Password, "mytoken") + } +} + +func TestToGitCmdAuth_Nil(t *testing.T) { + t.Parallel() + + if got := toGitCmdAuth(nil); got != nil { + t.Fatalf("toGitCmdAuth(nil) = %v, want nil", got) + } +} + +func TestToGitCmdAuth_FromTokenAuth_GitHubApp(t *testing.T) { + t.Parallel() + + repo := types.Repo{ + Token: "ghs_testtoken123", + NoTokenUser: true, + } + auth := tokenAuth(repo) + gitAuth := toGitCmdAuth(auth) + if gitAuth == nil { + t.Fatal("toGitCmdAuth returned nil, want *gitcmd.Auth") + } + if gitAuth.Username != "x-access-token" { + t.Errorf("Username = %q, want %q", gitAuth.Username, "x-access-token") + } + if gitAuth.Password != "ghs_testtoken123" { + t.Errorf("Password = %q, want %q", gitAuth.Password, "ghs_testtoken123") + } + + env := gitAuth.Env() + want := map[string]bool{ + "GICKUP_GIT_USERNAME=x-access-token": false, + "GICKUP_GIT_PASSWORD=ghs_testtoken123": false, + "GIT_CONFIG_KEY_1=credential.helper": false, + } + for _, e := range env { + if _, ok := want[e]; ok { + want[e] = true + } + if strings.Contains(e, "http.extraHeader") { + t.Errorf("unexpected http.extraHeader in env: %s", e) + } + } + for e, found := range want { + if !found { + t.Errorf("env missing %q: %v", e, env) + } + } +} + +func TestToGitCmdAuth_FromUsernamePassword(t *testing.T) { + t.Parallel() + + auth := &http.BasicAuth{ + Username: "myuser", + Password: "mypassword", + } + gitAuth := toGitCmdAuth(auth) + if gitAuth == nil { + t.Fatal("toGitCmdAuth returned nil, want *gitcmd.Auth") + } + if gitAuth.Username != "myuser" { + t.Errorf("Username = %q, want %q", gitAuth.Username, "myuser") + } + if gitAuth.Password != "mypassword" { + t.Errorf("Password = %q, want %q", gitAuth.Password, "mypassword") + } +} + +func TestToGitCmdAuth_SSHAuth(t *testing.T) { + t.Parallel() + + auth := &ssh.PublicKeys{} + if got := toGitCmdAuth(auth); got != nil { + t.Fatalf("toGitCmdAuth(ssh) = %v, want nil", got) + } +} + +func TestSanitizeRemote_RewritesCredentialURL(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + r, err := git.PlainInit(dir, true) + if err != nil { + t.Fatalf("failed to init repo: %v", err) + } + + _, err = r.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"https://xyz:expired-token-123@github.com/owner/repo.git"}, + }) + if err != nil { + t.Fatalf("failed to create remote: %v", err) + } + + err = sanitizeRemote(r, "https://github.com/owner/repo.git") + if err != nil { + t.Fatalf("sanitizeRemote returned error: %v", err) + } + + // Reopen the repository from disk to verify persistence + reopened, err := git.PlainOpen(dir) + if err != nil { + t.Fatalf("failed to reopen repo: %v", err) + } + + cfg, err := reopened.Config() + if err != nil { + t.Fatalf("failed to get config: %v", err) + } + + rem, ok := cfg.Remotes["origin"] + if !ok { + t.Fatal("remote origin not found") + } + + wantURLs := []string{"https://github.com/owner/repo.git"} + if !reflect.DeepEqual(rem.URLs, wantURLs) { + t.Errorf("remote URLs = %v, want %v", rem.URLs, wantURLs) + } + + for _, u := range rem.URLs { + if strings.Contains(u, "expired-token-123") || strings.Contains(u, "xyz:") || strings.Contains(u, "@") { + t.Errorf("token or credentials leaked in remote URL: %s", u) + } + } +} + +func TestSanitizeRemote_PreservesSSH(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + r, err := git.PlainInit(dir, true) + if err != nil { + t.Fatalf("failed to init repo: %v", err) + } + + _, err = r.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"git@github.com:owner/repo.git"}, + }) + if err != nil { + t.Fatalf("failed to create remote: %v", err) + } + + err = sanitizeRemote(r, "https://github.com/owner/repo.git") + if err != nil { + t.Fatalf("sanitizeRemote returned error: %v", err) + } + + cfg, err := r.Config() + if err != nil { + t.Fatalf("failed to get config: %v", err) + } + + rem := cfg.Remotes["origin"] + wantURLs := []string{"git@github.com:owner/repo.git"} + if !reflect.DeepEqual(rem.URLs, wantURLs) { + t.Errorf("SSH remote URLs = %v, want %v", rem.URLs, wantURLs) + } +} + +func TestSanitizeRemote_PreservesCleanURL(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + r, err := git.PlainInit(dir, true) + if err != nil { + t.Fatalf("failed to init repo: %v", err) + } + + _, err = r.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"https://github.com/owner/repo.git"}, + }) + if err != nil { + t.Fatalf("failed to create remote: %v", err) + } + + err = sanitizeRemote(r, "https://github.com/owner/repo.git") + if err != nil { + t.Fatalf("sanitizeRemote returned error: %v", err) + } + + cfg, err := r.Config() + if err != nil { + t.Fatalf("failed to get config: %v", err) + } + + rem := cfg.Remotes["origin"] + wantURLs := []string{"https://github.com/owner/repo.git"} + if !reflect.DeepEqual(rem.URLs, wantURLs) { + t.Errorf("clean remote URLs = %v, want %v", rem.URLs, wantURLs) + } +} + +func TestGetPushRefSpecs(t *testing.T) { + t.Parallel() + + repo, err := git.Init(memory.NewStorage(), nil) + if err != nil { + t.Fatalf("init repository: %v", err) + } + + hash := plumbing.NewHash("1111111111111111111111111111111111111111") + for _, name := range []plumbing.ReferenceName{ + plumbing.NewBranchReferenceName("master"), + plumbing.NewBranchReferenceName("feature"), + plumbing.NewTagReferenceName("v1.0.0"), + plumbing.NewRemoteReferenceName("origin", "feature"), + } { + if err := repo.Storer.SetReference(plumbing.NewHashReference(name, hash)); err != nil { + t.Fatalf("set reference %s: %v", name, err) + } + } + + refspecs, err := getPushRefSpecs(repo, plumbing.NewBranchReferenceName("master")) + if err != nil { + t.Fatalf("getPushRefSpecs() error: %v", err) + } + + want := map[string]bool{ + "refs/heads/feature:refs/heads/feature": false, + "refs/tags/v1.0.0:refs/tags/v1.0.0": false, + } + for _, refspec := range refspecs { + key := refspec.String() + if _, ok := want[key]; !ok { + t.Fatalf("unexpected refspec %q", key) + } + want[key] = true + } + for refspec, found := range want { + if !found { + t.Errorf("missing refspec %q", refspec) + } + } +} diff --git a/main.go b/main.go index fd35426..cc55b8a 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "reflect" "strconv" "strings" + "sync" "time" "github.com/alecthomas/kong" @@ -20,6 +21,7 @@ import ( "github.com/cooperspencer/gickup/github" "github.com/cooperspencer/gickup/gitlab" "github.com/cooperspencer/gickup/gogs" + "github.com/cooperspencer/gickup/httpclient" "github.com/cooperspencer/gickup/local" "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/metrics/apprise" @@ -28,9 +30,13 @@ import ( "github.com/cooperspencer/gickup/metrics/ntfy" "github.com/cooperspencer/gickup/metrics/prometheus" "github.com/cooperspencer/gickup/onedev" + "github.com/cooperspencer/gickup/opengist" + "github.com/cooperspencer/gickup/radicle" "github.com/cooperspencer/gickup/s3" "github.com/cooperspencer/gickup/sourcehut" "github.com/cooperspencer/gickup/types" + "github.com/cooperspencer/gickup/webdav" + "github.com/cooperspencer/gickup/webui" "github.com/cooperspencer/gickup/whatever" "github.com/cooperspencer/gickup/zip" "github.com/go-git/go-git/v5" @@ -44,6 +50,7 @@ import ( ) var cli struct { + WebUI string `flag name:"webui" placeholder:"ADDR" help:"Start the web interface at this address, even without a configuration file (e.g. :8080; overrides webui.addr)."` Configfiles []string `arg name:"conf" help:"Path to the configfile." default:"conf.yml"` Version bool `flag name:"version" help:"Show version."` Dry bool `flag name:"dryrun" help:"Make a dry-run."` @@ -55,14 +62,20 @@ var cli struct { var version = "unknown" func readConfigFile(configfile string) []*types.Conf { + confs, err := loadConfigFile(configfile) + if err != nil { + log.Fatal().Err(err).Str("file", configfile).Msg("Cannot read configuration") + } + return confs +} + +func loadConfigFile(configfile string) ([]*types.Conf, error) { conf := []*types.Conf{} - cfgdata, err := os.Open(configfile) + cfgdata, err := os.Open(filepath.Clean(configfile)) if err != nil { - log.Fatal(). - Str("stage", "readconfig"). - Str("file", configfile). - Msgf("Cannot open config file from %s", types.Red(configfile)) + return nil, err } + defer cfgdata.Close() dec := yaml.NewDecoder(cfgdata) @@ -74,26 +87,14 @@ func readConfigFile(configfile string) []*types.Conf { if errors.Is(err, io.EOF) { break } else if err != nil { - if len(conf) > 0 { - log.Fatal(). - Str("stage", "readconfig"). - Str("file", configfile). - Msgf("an error occurred in the %d place of %s", i, configfile) - } else { - log.Fatal(). - Str("stage", "readconfig"). - Str("file", configfile). - Msg(err.Error()) - } + return nil, fmt.Errorf("config document %d: %w", i+1, err) } if reflect.ValueOf(c).IsZero() { continue } - for i, local := range c.Destination.Local { - c.Destination.Local[i].Path = substituteHomeForTildeInPath(local.Path) - } + expandConfigPaths(&c) if !reflect.ValueOf(c).IsZero() { if len(conf) > 0 { @@ -106,7 +107,40 @@ func readConfigFile(configfile string) []*types.Conf { } } - return conf + return conf, nil +} + +func expandConfigPaths(c *types.Conf) { + c.Log.FileLogging.Dir = substituteHomeForTildeInPath(c.Log.FileLogging.Dir) + + expandGenRepoPaths(c.Source.Gogs) + expandGenRepoPaths(c.Source.Gitlab) + expandGenRepoPaths(c.Source.Github) + expandGenRepoPaths(c.Source.Gitea) + expandGenRepoPaths(c.Source.BitBucket) + expandGenRepoPaths(c.Source.OneDev) + expandGenRepoPaths(c.Source.Sourcehut) + expandGenRepoPaths(c.Source.Opengist) + expandGenRepoPaths(c.Source.Any) + + expandGenRepoPaths(c.Destination.Gitlab) + expandGenRepoPaths(c.Destination.Github) + expandGenRepoPaths(c.Destination.Gitea) + expandGenRepoPaths(c.Destination.Gogs) + expandGenRepoPaths(c.Destination.OneDev) + expandGenRepoPaths(c.Destination.Sourcehut) + + for i, local := range c.Destination.Local { + c.Destination.Local[i].Path = substituteHomeForTildeInPath(local.Path) + } +} + +func expandGenRepoPaths(repos []types.GenRepo) { + for i := range repos { + repos[i].TokenFile = substituteHomeForTildeInPath(repos[i].TokenFile) + repos[i].SSHKey = substituteHomeForTildeInPath(repos[i].SSHKey) + repos[i].AppPrivateKeyFile = substituteHomeForTildeInPath(repos[i].AppPrivateKeyFile) + } } func getUserHome() (string, error) { @@ -185,6 +219,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "local", d.Path).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("local").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "local", + DestAddr: d.Path, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } for _, d := range conf.Destination.S3 { @@ -220,6 +265,7 @@ func backup(repos []types.Repo, conf *types.Conf) { } defer os.RemoveAll(tempdir) + r.Origin.LFS = d.LFS tempClonePath := path.Join(tempdir, r.Name) _, err = local.TempCloneBare(r, tempClonePath) if err != nil { @@ -245,20 +291,22 @@ func backup(repos []types.Repo, conf *types.Conf) { } } - // Check if environment variables are used for accesskey and secretkey - d.AccessKey, err = d.GetKey(d.AccessKey) - if err != nil { - log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) - } - d.SecretKey, err = d.GetKey(d.SecretKey) - if err != nil { - log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) - } - if d.Token != "" { - d.Token, err = d.GetKey(d.Token) + if d.UseStaticCreds { + // Check if environment variables are used for accesskey and secretkey + d.AccessKey, err = d.GetKey(d.AccessKey) if err != nil { log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) } + d.SecretKey, err = d.GetKey(d.SecretKey) + if err != nil { + log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) + } + if d.Token != "" { + d.Token, err = d.GetKey(d.Token) + if err != nil { + log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) + } + } } if d.Zip { @@ -296,6 +344,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "s3", d.Endpoint).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("s3").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "s3", + DestAddr: d.Endpoint, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -387,6 +446,114 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "azureblob", d.Container).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("azureblob").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "azureblob", + DestAddr: d.Container, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) + } + } + + for _, d := range conf.Destination.WebDAV { + repotime := time.Now() + status := 0 + + logOp := "uploading" + if d.Zip { + logOp = "zipping and uploading" + } + log.Info(). + Str("stage", "webdav"). + Str("url", d.Url). + Msgf("%s %s to %s", logOp, types.Blue(r.Name), d.Url) + + if !cli.Dry { + func() { + defer func() { + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, RepoName: r.Name, RepoURL: r.URL, + Owner: r.Owner, Hoster: r.Hoster, DestType: "webdav", + DestAddr: d.Url, Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) + }() + tempname := fmt.Sprintf("webdav-%x", repotime) + tempdir, err := os.MkdirTemp(os.TempDir(), tempname) + if err != nil { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Msg(err.Error()) + return + } + + if d.Structured { + r.Name = path.Join(r.Hoster, r.Owner, r.Name) + } + + if d.DateCreateDir { + r.Name = currentDateDir + r.Name + } + + defer os.RemoveAll(tempdir) + tempClonePath := path.Join(tempdir, r.Name) + _, err = local.TempCloneBare(r, tempClonePath) + if err != nil { + switch { + case errors.Is(err, git.NoErrAlreadyUpToDate): + log.Info(). + Str("stage", "webdav"). + Msg(err.Error()) + case errors.Is(err, transport.ErrEmptyRemoteRepository): + log.Warn(). + Str("repo", r.Name). + Msgf("%s - Skipping backup", err.Error()) + return + default: + log.Error(). + Str("stage", "tempclone"). + Str("git", "clone"). + Msg(err.Error()) + os.RemoveAll(tempdir) + return + } + } + + if d.Zip { + log.Info(). + Msgf("zipping %s", types.Green(r.Name)) + err := zip.Zip(tempClonePath, []string{tempClonePath}) + if err != nil { + log.Error(). + Str("stage", "zip"). + Str("repo", r.Name). + Msg(err.Error()) + log.Error().Msgf("Skipping backup of %s due to error while zipping", r.Name) + return + } + } + err = webdav.UploadDirToWebDAV(tempdir, d) + if err != nil { + log.Error().Str("stage", "webdav").Str("url", d.Url).Msg(err.Error()) + return + } + err = webdav.DeleteObjectsNotInRepo(tempdir, r.Name, d) + if err != nil { + log.Error().Str("stage", "webdav").Str("url", d.Url).Msg(err.Error()) + return + } + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "webdav", d.Url).Set(time.Since(repotime).Seconds()) + status = 1 + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "webdav", d.Url).Set(float64(status)) + prometheus.DestinationBackupsComplete.WithLabelValues("webdav").Inc() + }() } } @@ -445,7 +612,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, d.GetToken(), r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -475,6 +642,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "gitea", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -527,7 +705,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, d.GetToken(), r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -557,6 +735,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gogs").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "gogs", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -613,7 +802,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, d.GetToken(), r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -643,6 +832,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitlab", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gitlab").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "gitlab", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -685,7 +885,7 @@ func backup(repos []types.Repo, conf *types.Conf) { } } - cloneurl, err := github.GetOrCreate(d, r) + cloneurl, destinationtoken, err := github.GetOrCreate(d, r) if err != nil { log.Error(). Str("stage", "github"). @@ -695,7 +895,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, destinationtoken, r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -718,6 +918,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "github", "https://github.com").Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("github").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "github", + DestAddr: "https://github.com", + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } } @@ -773,7 +984,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, d.GetToken(), r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -796,6 +1007,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "onedev", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("onedev").Inc() os.RemoveAll(tempdir) + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "onedev", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } } @@ -852,7 +1074,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, d.GetToken(), r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -875,15 +1097,99 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "sourcehut", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("sourcehut").Inc() os.RemoveAll(tempdir) + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "sourcehut", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } } + for _, d := range conf.Destination.Radicle { + repotime := time.Now() + status := 0 + + radhome, _ := radicle.Home() + + log.Info(). + Str("stage", "radicle"). + Str("home", radhome). + Msgf("mirroring %s to %s", types.Blue(r.Name), radhome) + + if !cli.Dry { + func() { + defer func() { + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, RepoName: r.Name, RepoURL: r.URL, + Owner: r.Owner, Hoster: r.Hoster, DestType: "radicle", + DestAddr: radhome, Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) + }() + tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("radicle-%x", repotime)) + if err != nil { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Msg(err.Error()) + return + } + + defer os.RemoveAll(tempdir) + _, err = local.TempClone(r, tempdir) + if err != nil { + if errors.Is(err, git.NoErrAlreadyUpToDate) { + log.Info(). + Str("stage", "radicle"). + Str("url", r.URL). + Msg(err.Error()) + } else { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Str("git", "clone"). + Msg(err.Error()) + return + } + } + + rid, err := radicle.Mirror(r, d, tempdir) + if err != nil { + log.Error(). + Str("stage", "radicle"). + Str("url", r.URL). + Msg(err.Error()) + return + } + + log.Info(). + Str("stage", "radicle"). + Str("rid", fmt.Sprintf("rad:%s", rid)). + Msgf("mirrored %s", types.Green(r.Name)) + + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "radicle", radhome).Set(time.Since(repotime).Seconds()) + status = 1 + }() + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "radicle", radhome).Set(float64(status)) + prometheus.DestinationBackupsComplete.WithLabelValues("radicle").Inc() + } + } + prometheus.SourceBackupsComplete.WithLabelValues(r.Name).Inc() } } func runBackup(conf *types.Conf, num int) { + webui.Global.BeginRun() + defer webui.Global.EndRun() log.Info().Msg("Backup run starting") numstring := strconv.Itoa(num) @@ -892,6 +1198,8 @@ func runBackup(conf *types.Conf, num int) { prometheus.JobsStarted.Inc() + httpclient.ApplyGickupUserAgent() + // Github repos, ran := github.Get(conf) if ran { @@ -944,20 +1252,36 @@ func runBackup(conf *types.Conf, num int) { } backup(repos, conf) + repos, ran = opengist.Get(conf) + if ran { + prometheus.CountReposDiscovered.WithLabelValues("opengist", numstring).Set(float64(len(repos))) + } + backup(repos, conf) + endTime := time.Now() duration := endTime.Sub(startTime) prometheus.JobsComplete.Inc() prometheus.JobDuration.Observe(duration.Seconds()) - if len(conf.Metrics.Heartbeat.URLs) > 0 { - heartbeat.Send(conf.Metrics.Heartbeat) + exitCode := logger.GetExitCode() + success := exitCode == 0 + + title := "Backup done" + msg := fmt.Sprintf("backup took %v", duration) + if !success { + title = "Backup failed" + msg = fmt.Sprintf("backup took %v, check the logs for errors", duration) + } + + if len(conf.Metrics.Heartbeat.URLs) > 0 || len(conf.Metrics.Heartbeat.FailureURLs) > 0 { + heartbeat.Send(conf.Metrics.Heartbeat, success) } if len(conf.Metrics.PushConfigs.Ntfy) > 0 { for _, pusher := range conf.Metrics.PushConfigs.Ntfy { pusher.ResolveToken() - err := ntfy.Notify(fmt.Sprintf("backup took %v", duration), *pusher) + err := ntfy.Notify(title, msg, *pusher) if err != nil { log.Warn().Str("push", "ntfy").Err(err).Msg("couldn't send message") } @@ -967,7 +1291,7 @@ func runBackup(conf *types.Conf, num int) { if len(conf.Metrics.PushConfigs.Gotify) > 0 { for _, pusher := range conf.Metrics.PushConfigs.Gotify { pusher.ResolveToken() - err := gotify.Notify(fmt.Sprintf("backup took %v", duration), *pusher) + err := gotify.Notify(title, msg, *pusher) if err != nil { log.Warn().Str("push", "gotify").Err(err).Msg("couldn't send message") } @@ -976,14 +1300,14 @@ func runBackup(conf *types.Conf, num int) { if len(conf.Metrics.PushConfigs.Apprise) > 0 { for _, pusher := range conf.Metrics.PushConfigs.Apprise { - err := apprise.Notify(fmt.Sprintf("backup took %v", duration), *pusher) + err := apprise.Notify(title, msg, *pusher) if err != nil { log.Warn().Str("push", "apprise").Err(err).Msg("couldn't send message") } } } - exitCode := logger.GetExitCode() - if exitCode != 0 { + + if !success { log.Warn().Msgf("Encountered at least one error during the run. Check the logs. Exiting with status=%d", exitCode) } @@ -999,11 +1323,25 @@ func runBackup(conf *types.Conf, num int) { func playsForever(c *cron.Cron, conffiles []string, confs []*types.Conf) bool { for { checkconfigs := []*types.Conf{} + valid := true for _, f := range conffiles { - checkconfigs = append(checkconfigs, readConfigFile(f)...) + loaded, err := loadConfigFile(f) + if errors.Is(err, os.ErrNotExist) && cli.WebUI != "" { + continue + } + if err != nil { + log.Error().Err(err).Str("file", f).Msg("Cannot reload configuration; keeping current configuration") + valid = false + break + } + checkconfigs = append(checkconfigs, loaded...) + } + if !valid { + time.Sleep(5 * time.Second) + continue } - if checkconfigs[0].HasValidCronSpec() { + if len(checkconfigs) > 0 && checkconfigs[0].HasValidCronSpec() { for num, config := range checkconfigs { if !config.HasValidCronSpec() { checkconfigs[num].Cron = checkconfigs[0].Cron @@ -1014,8 +1352,10 @@ func playsForever(c *cron.Cron, conffiles []string, confs []*types.Conf) bool { if !cmp.Equal(confs, checkconfigs) { log.Info().Msg("config changed") log.Debug().Msg(cmp.Diff(confs, checkconfigs)) - for _, entry := range c.Entries() { - c.Remove(entry.ID) + if c != nil { + for _, entry := range c.Entries() { + c.Remove(entry.ID) + } } return true } @@ -1064,10 +1404,17 @@ func main() { Msgf("this is a %s", types.Blue("dry run")) } + var webuiOnce sync.Once + var webuiAddr string + if cli.WebUI != "" { + webuiAddr = cli.WebUI + } + init := true for { reload := false confs := []*types.Conf{} + confNames := []string{} for i, f := range cli.Configfiles { log.Info().Str("file", f). Msgf("Reading %s", types.Green(f)) @@ -1076,7 +1423,41 @@ func main() { log.Panic().Err(err).Msgf("there is an issue with %s", f) } cli.Configfiles[i] = absf - confs = append(confs, readConfigFile(absf)...) + base := filepath.Base(absf) + before := len(confs) + loaded, err := loadConfigFile(absf) + if err != nil { + if cli.WebUI == "" { + log.Fatal().Err(err).Str("file", absf).Msg("Cannot read configuration") + } + if !errors.Is(err, os.ErrNotExist) { + log.Error().Err(err).Str("file", absf).Msg("Repair configuration in the web interface") + } + } else { + confs = append(confs, loaded...) + } + added := len(confs) - before + for j := range added { + if added > 1 { + confNames = append(confNames, fmt.Sprintf("%s #%d", base, j+1)) + } else { + confNames = append(confNames, base) + } + } + } + + if cli.WebUI != "" { + webui.Global.SetConfigFiles(cli.Configfiles) + webuiOnce.Do(func() { go webui.Serve(webuiAddr) }) + } + if len(confs) == 0 { + if cli.WebUI == "" { + log.Fatal().Msg("No configuration found; use --webui :8080 to create one in the browser") + } + webui.Global.SetConfigs(nil) + webui.Global.SetRunFunc(nil) + time.Sleep(5 * time.Second) + continue } logConf := confs[0].Log @@ -1087,6 +1468,46 @@ func main() { log.Logger = logger.CreateLogger(logConf) + // Determine webUI address from config file. + if webuiAddr == "" && confs[0].WebUI.Addr != "" { + webuiAddr = confs[0].WebUI.Addr + } + + // Register config summaries and run callback; start server once. + if webuiAddr != "" { + infos := make([]webui.ConfigInfo, len(confs)) + for i, c := range confs { + info := webui.ConfigInfo{ + Index: i, + Name: confNames[i], + Sources: c.Source.Count(), + Dests: c.Destination.Count(), + } + if c.HasValidCronSpec() { + info.CronSpec = c.Cron + if next, err := c.GetNextRun(); err == nil { + info.NextRun = next.Format(time.RFC3339) + } + } + infos[i] = info + } + webui.Global.SetConfigFiles(cli.Configfiles) + webui.Global.SetConfigs(infos) + currentConfs := confs + webui.Global.SetRunFunc(func(idx int) { + if idx < 0 { + for i, c := range currentConfs { + runBackup(c, i) + } + } else if idx < len(currentConfs) { + runBackup(currentConfs[idx], idx) + } + }) + webuiOnce.Do(func() { + go webui.Serve(webuiAddr) + }) + } + validcron := confs[0].HasValidCronSpec() var c *cron.Cron @@ -1145,6 +1566,12 @@ func main() { } reload = playsForever(c, cli.Configfiles, confs) log.Info().Msg("reloading config...") + } else if webuiAddr != "" { + // No cron but web UI is active: stay alive and watch for config changes. + reload = playsForever(nil, cli.Configfiles, confs) + if reload { + log.Info().Msg("reloading config...") + } } if !reload { break diff --git a/main_github_test.go b/main_github_test.go new file mode 100644 index 0000000..3616f44 --- /dev/null +++ b/main_github_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/cooperspencer/gickup/types" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" +) + +//nolint:paralleltest // backup calls providers that assign package-level loggers. +func TestBackupGitHubAppPushCredential(t *testing.T) { + const wantToken = "installation-token-for-test" + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + keyFile := filepath.Join(t.TempDir(), "app.pem") + if err := os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key), + }), 0o600); err != nil { + t.Fatal(err) + } + + sourceDir := t.TempDir() + source, err := git.PlainInit(sourceDir, false) + if err != nil { + t.Fatal(err) + } + worktree, err := source.Worktree() + if err != nil { + t.Fatal(err) + } + if _, err := worktree.Commit("seed", &git.CommitOptions{ + AllowEmptyCommits: true, + Author: &object.Signature{Name: "test", Email: "test@example.com", When: time.Now()}, + }); err != nil { + t.Fatal(err) + } + + var passwordsMu sync.Mutex + var passwords []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "POST /api/v3/app/installations/42/access_tokens": + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]string{ + "token": wantToken, "expires_at": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), + }) + case "GET /api/v3/orgs/example": + _, _ = w.Write([]byte(`{"login":"example"}`)) + case "GET /api/v3/repos/example/repo": + _ = json.NewEncoder(w).Encode(map[string]string{"clone_url": "http://" + r.Host + "/example/repo.git"}) + case "GET /example/repo.git/info/refs": + _, password, _ := r.BasicAuth() + passwordsMu.Lock() + passwords = append(passwords, password) + passwordsMu.Unlock() + w.WriteHeader(http.StatusUnauthorized) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + backup([]types.Repo{{Name: "repo", URL: sourceDir}}, &types.Conf{ + Destination: types.Destination{Github: []types.GenRepo{{ + URL: server.URL, Organization: "example", + AppID: 1, AppInstallationID: 42, AppPrivateKeyFile: keyFile, + }}}, + }) + passwordsMu.Lock() + gotPasswords := passwords + passwords = nil + passwordsMu.Unlock() + if len(gotPasswords) == 0 { + t.Fatal("backup did not attempt a push") + } + for _, got := range gotPasswords { + if got != wantToken { + t.Fatalf("backup push password = %q, want resolved installation token", got) + } + } +} diff --git a/main_test.go b/main_test.go index 46486a9..8435301 100644 --- a/main_test.go +++ b/main_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "path/filepath" "strings" "testing" ) @@ -40,7 +39,6 @@ func TestTildeReplacement_TildeDir(t *testing.T) { func TestReadConfigFile_InheritsPushConfigsAndExpandsHome(t *testing.T) { t.Parallel() - configPath := filepath.Join(t.TempDir(), "conf.yml") config := `destination: local: - path: "~/primary" @@ -54,10 +52,15 @@ destination: local: - path: "/tmp/secondary" ` - - if err := os.WriteFile(configPath, []byte(config), 0o644); err != nil { + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { t.Fatalf("write config: %v", err) } + f.Close() confs := readConfigFile(configPath) if len(confs) != 2 { @@ -80,3 +83,230 @@ destination: t.Fatal("expected initial push config to be populated") } } + +func TestReadConfigFile_ExpandsHomeInFileBackedConfigKeys(t *testing.T) { + t.Parallel() + + config := `source: + github: + - token_file: "~/github-token" + sshkey: "~/github-ssh" + app_private_key_file: "~/github-app.pem" +destination: + gitea: + - token_file: "~/gitea-token" + sshkey: "~/gitea-ssh" + app_private_key_file: "~/gitea-app.pem" +log: + file-logging: + dir: "~/gickup-logs" + file: "gickup.log" +` + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { + t.Fatalf("write config: %v", err) + } + f.Close() + + confs := readConfigFile(configPath) + if len(confs) != 1 { + t.Fatalf("expected 1 config, got %d", len(confs)) + } + + source := confs[0].Source.Github[0] + for name, path := range map[string]string{ + "source token_file": source.TokenFile, + "source sshkey": source.SSHKey, + "source app_private_key_file": source.AppPrivateKeyFile, + } { + if strings.HasPrefix(path, "~") { + t.Fatalf("%s was not expanded: %q", name, path) + } + } + + destination := confs[0].Destination.Gitea[0] + for name, path := range map[string]string{ + "destination token_file": destination.TokenFile, + "destination sshkey": destination.SSHKey, + "destination app_private_key_file": destination.AppPrivateKeyFile, + "log file-logging dir": confs[0].Log.FileLogging.Dir, + } { + if strings.HasPrefix(path, "~") { + t.Fatalf("%s was not expanded: %q", name, path) + } + } +} + +func TestReadConfigFile_S3UseStaticCredsTrue(t *testing.T) { + t.Parallel() + + config := `destination: + s3: + - bucket: "my-bucket" + endpoint: "s3.example.com" + use_static_creds: true + accesskey: "AKID" + secretkey: "SECRET" +` + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { + t.Fatalf("write config: %v", err) + } + f.Close() + + confs := readConfigFile(configPath) + if len(confs) != 1 { + t.Fatalf("expected 1 config, got %d", len(confs)) + } + + s3 := confs[0].Destination.S3 + if len(s3) != 1 { + t.Fatalf("expected 1 S3 destination, got %d", len(s3)) + } + + if !s3[0].UseStaticCreds { + t.Fatal("expected UseStaticCreds to be true") + } +} + +func TestReadConfigFile_S3UseStaticCredsFalse(t *testing.T) { + t.Parallel() + + config := `destination: + s3: + - bucket: "my-bucket" + endpoint: "s3.example.com" + use_static_creds: false +` + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { + t.Fatalf("write config: %v", err) + } + f.Close() + + confs := readConfigFile(configPath) + if len(confs) != 1 { + t.Fatalf("expected 1 config, got %d", len(confs)) + } + + s3 := confs[0].Destination.S3 + if len(s3) != 1 { + t.Fatalf("expected 1 S3 destination, got %d", len(s3)) + } + + if s3[0].UseStaticCreds { + t.Fatal("expected UseStaticCreds to be false") + } +} + +func TestReadConfigFile_S3UseStaticCredsResolvesEnvVars(t *testing.T) { + t.Setenv("TEST_S3_ACCESS", "resolved-access") + t.Setenv("TEST_S3_SECRET", "resolved-secret") + + config := `destination: + s3: + - bucket: "my-bucket" + endpoint: "s3.example.com" + use_static_creds: true + accesskey: "TEST_S3_ACCESS" + secretkey: "TEST_S3_SECRET" +` + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { + t.Fatalf("write config: %v", err) + } + f.Close() + + confs := readConfigFile(configPath) + s3 := confs[0].Destination.S3[0] + + // Simulate the key resolution that backup() does when UseStaticCreds is true + if !s3.UseStaticCreds { + t.Fatal("expected UseStaticCreds to be true") + } + + accessKey, err := s3.GetKey(s3.AccessKey) + if err != nil { + t.Fatalf("GetKey(accesskey) error = %v", err) + } + + secretKey, err := s3.GetKey(s3.SecretKey) + if err != nil { + t.Fatalf("GetKey(secretkey) error = %v", err) + } + + if accessKey != "resolved-access" { + t.Fatalf("accesskey = %q, want resolved-access", accessKey) + } + + if secretKey != "resolved-secret" { + t.Fatalf("secretkey = %q, want resolved-secret", secretKey) + } +} + +func TestReadConfigFile_S3UseStaticCredsAbsentSkipsKeyResolution(t *testing.T) { + t.Parallel() + + config := `destination: + s3: + - bucket: "my-bucket" + endpoint: "s3.example.com" +` + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { + t.Fatalf("write config: %v", err) + } + f.Close() + + confs := readConfigFile(configPath) + s3 := confs[0].Destination.S3[0] + + // When UseStaticCreds is false (zero value), backup() skips key resolution entirely + if s3.UseStaticCreds { + t.Fatal("expected UseStaticCreds to be false when absent from config") + } +} + +func TestLoadConfigFileSetupStates(t *testing.T) { + t.Parallel() + + file := t.TempDir() + "/conf.yml" + if _, err := loadConfigFile(file); !os.IsNotExist(err) { + t.Fatalf("missing config: %v", err) + } + for _, content := range []string{"", "# empty config\n", "{}"} { + if err := os.WriteFile(file, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + confs, err := loadConfigFile(file) + if err != nil || len(confs) != 0 { + t.Fatalf("empty config: %v, %v", confs, err) + } + } + if err := os.WriteFile(file, []byte("source: ["), 0o600); err != nil { + t.Fatal(err) + } + if _, err := loadConfigFile(file); err == nil { + t.Fatal("invalid configuration accepted") + } +} diff --git a/metrics/apprise/apprise.go b/metrics/apprise/apprise.go index 042d095..571ea46 100644 --- a/metrics/apprise/apprise.go +++ b/metrics/apprise/apprise.go @@ -12,20 +12,22 @@ import ( ) type Request struct { - Body string `json:"body"` - Tags []string `json:"tags,omitempty"` - Urls []string `json:"urls,omitempty"` + Title string `json:"title,omitempty"` + Body string `json:"body"` + Tags []string `json:"tags,omitempty"` + Urls []string `json:"urls,omitempty"` } type ErrorMsg struct { Error string `json:"error"` } -func Notify(msg string, config types.AppriseConfig) error { +func Notify(title, msg string, config types.AppriseConfig) error { payload := Request{ - Body: msg, - Urls: config.Urls, - Tags: config.Tags, + Title: title, + Body: msg, + Urls: config.Urls, + Tags: config.Tags, } jsonData, err := json.Marshal(payload) diff --git a/metrics/apprise/apprise_test.go b/metrics/apprise/apprise_test.go index 002e787..1726f93 100644 --- a/metrics/apprise/apprise_test.go +++ b/metrics/apprise/apprise_test.go @@ -42,7 +42,7 @@ func TestNotifySendsExpectedRequest(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.AppriseConfig{ + err := Notify("Backup done", "backup complete", types.AppriseConfig{ Url: server.URL, Config: "team", Tags: []string{"backup", "nightly"}, @@ -61,7 +61,7 @@ func TestNotifyReturnsAPIError(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.AppriseConfig{Url: server.URL}) + err := Notify("Backup done", "backup complete", types.AppriseConfig{Url: server.URL}) if err == nil { t.Fatal("expected apprise error") } diff --git a/metrics/gotify/gotify.go b/metrics/gotify/gotify.go index 07a0ae9..6cbdbb8 100644 --- a/metrics/gotify/gotify.go +++ b/metrics/gotify/gotify.go @@ -11,7 +11,7 @@ import ( "github.com/cooperspencer/gickup/types" ) -func Notify(msg string, config types.PushConfig) error { +func Notify(title, msg string, config types.PushConfig) error { if !strings.HasSuffix(config.Url, "/") { config.Url += "/" } @@ -20,7 +20,7 @@ func Notify(msg string, config types.PushConfig) error { payload := map[string]string{} payload["message"] = msg - payload["title"] = "Backup done" + payload["title"] = title body, err := json.Marshal(payload) if err != nil { diff --git a/metrics/gotify/gotify_test.go b/metrics/gotify/gotify_test.go index 71ab9cf..a08b3ec 100644 --- a/metrics/gotify/gotify_test.go +++ b/metrics/gotify/gotify_test.go @@ -38,7 +38,7 @@ func TestNotifySendsExpectedPayload(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) + err := Notify("Backup done", "backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) if err != nil { t.Fatalf("Notify() error = %v", err) } @@ -52,7 +52,7 @@ func TestNotifyReturnsStatusError(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) + err := Notify("Backup done", "backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) if err == nil { t.Fatal("expected status error") } diff --git a/metrics/heartbeat/heartbeat.go b/metrics/heartbeat/heartbeat.go index 7418bda..2b6f7ba 100644 --- a/metrics/heartbeat/heartbeat.go +++ b/metrics/heartbeat/heartbeat.go @@ -7,9 +7,21 @@ import ( "github.com/rs/zerolog/log" ) -func Send(conf types.HeartbeatConfig) { - for _, u := range conf.URLs { - log.Info().Str("url", u).Msg("sending heartbeat") +// Send pings the configured heartbeat URLs. +// +// On a successful backup run, conf.URLs is pinged. On a failed run, +// conf.FailureURLs is pinged instead, so monitoring services such as +// healthchecks.io that expose a dedicated "failure" endpoint can be +// notified immediately instead of only alerting once the regular +// heartbeat times out. +func Send(conf types.HeartbeatConfig, success bool) { + urls := conf.URLs + if !success { + urls = conf.FailureURLs + } + + for _, u := range urls { + log.Info().Str("url", u).Bool("success", success).Msg("sending heartbeat") resp, err := http.Get(u) //nolint:noctx if err != nil { log.Error().Str("monitoring", "heartbeat").Msg(err.Error()) diff --git a/metrics/heartbeat/heartbeat_test.go b/metrics/heartbeat/heartbeat_test.go index 2781d6c..f6f05d5 100644 --- a/metrics/heartbeat/heartbeat_test.go +++ b/metrics/heartbeat/heartbeat_test.go @@ -20,9 +20,39 @@ func TestSendCallsEachHeartbeatURL(t *testing.T) { })) defer server.Close() - Send(types.HeartbeatConfig{URLs: []string{server.URL, server.URL}}) + Send(types.HeartbeatConfig{URLs: []string{server.URL, server.URL}}, true) if got := atomic.LoadInt32(&hits); got != 2 { t.Fatalf("hits = %d, want 2", got) } } + +func TestSendCallsFailureURLsOnFailure(t *testing.T) { + t.Parallel() + + var successHits, failureHits int32 + + successServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + atomic.AddInt32(&successHits, 1) + w.WriteHeader(http.StatusNoContent) + })) + defer successServer.Close() + + failureServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + atomic.AddInt32(&failureHits, 1) + w.WriteHeader(http.StatusNoContent) + })) + defer failureServer.Close() + + Send(types.HeartbeatConfig{ + URLs: []string{successServer.URL}, + FailureURLs: []string{failureServer.URL}, + }, false) + + if got := atomic.LoadInt32(&successHits); got != 0 { + t.Fatalf("successHits = %d, want 0", got) + } + if got := atomic.LoadInt32(&failureHits); got != 1 { + t.Fatalf("failureHits = %d, want 1", got) + } +} diff --git a/metrics/ntfy/ntfy.go b/metrics/ntfy/ntfy.go index 6c9437c..7bb3c06 100644 --- a/metrics/ntfy/ntfy.go +++ b/metrics/ntfy/ntfy.go @@ -10,7 +10,7 @@ import ( "github.com/cooperspencer/gickup/types" ) -func Notify(msg string, config types.PushConfig) error { +func Notify(title, msg string, config types.PushConfig) error { url := config.Url payload := strings.NewReader(msg) @@ -21,7 +21,7 @@ func Notify(msg string, config types.PushConfig) error { } req.Header.Add("Content-Type", "text/plain") - req.Header.Add("Title", "Backup done") + req.Header.Add("Title", title) if config.Email != "" { req.Header.Add("Email", config.Email) } diff --git a/metrics/ntfy/ntfy_test.go b/metrics/ntfy/ntfy_test.go index 145803c..0d5b287 100644 --- a/metrics/ntfy/ntfy_test.go +++ b/metrics/ntfy/ntfy_test.go @@ -47,7 +47,7 @@ func TestNotifyAddsEmailHeader(t *testing.T) { })) defer server.Close() - err := Notify("backup took 2m", types.PushConfig{ + err := Notify("Backup done", "backup took 2m", types.PushConfig{ Url: server.URL, Token: "my-token", Email: "user@example.com", @@ -78,7 +78,7 @@ func TestNotifyUsesBasicAuth(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.PushConfig{Url: server.URL, User: "andy", Password: "password"}) + err := Notify("Backup done", "backup complete", types.PushConfig{Url: server.URL, User: "andy", Password: "password"}) if err != nil { t.Fatalf("Notify() error = %v", err) } @@ -87,7 +87,7 @@ func TestNotifyUsesBasicAuth(t *testing.T) { func TestNotifyRequiresCredentials(t *testing.T) { t.Parallel() - err := Notify("backup complete", types.PushConfig{Url: "https://ntfy.sh/topic"}) + err := Notify("Backup done", "backup complete", types.PushConfig{Url: "https://ntfy.sh/topic"}) if err == nil { t.Fatal("expected credential error") } @@ -105,7 +105,7 @@ func TestNotifyReturnsStatusError(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) + err := Notify("Backup done", "backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) if err == nil { t.Fatal("expected status error") } diff --git a/onedev/onedev.go b/onedev/onedev.go index 098c7b7..1632aa9 100644 --- a/onedev/onedev.go +++ b/onedev/onedev.go @@ -95,13 +95,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } } + if exclude[r.Name] { + continue + } + if len(repo.Include) > 0 { if !include[r.Name] { continue } - if exclude[r.Name] { - continue - } } defaultbranch, _, err := client.GetDefaultBranch(r.ID) diff --git a/opengist/opengist.go b/opengist/opengist.go new file mode 100644 index 0000000..ffa6e6f --- /dev/null +++ b/opengist/opengist.go @@ -0,0 +1,224 @@ +package opengist + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/cooperspencer/gickup/logger" + "github.com/cooperspencer/gickup/types" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +var sub zerolog.Logger + +// perPage is the page size used when paginating the Opengist API. +const perPage = 50 + +// gist mirrors the relevant fields of Opengist's GistSimple API object +// (internal/web/handlers/api/v1/types/gist.go upstream). +type gist struct { + ID string `json:"id"` + SlugURL string `json:"slug_url"` + Owner user `json:"owner"` + Title string `json:"title"` + Description string `json:"description"` + Public bool `json:"public"` + Visibility string `json:"visibility"` + CloneURL string `json:"clone_url"` + SSHURL string `json:"ssh_url"` +} + +// user mirrors the fields of Opengist's SimpleUser/PrivateUser. +type user struct { + Username string `json:"username"` +} + +// client is a minimal Opengist REST client for the API (Bearer token auth) over net/http. +type client struct { + base string + token string + http *http.Client +} + +func (c *client) get(path string, query url.Values) ([]byte, error) { + endpoint := c.base + path + if len(query) > 0 { + endpoint += "?" + query.Encode() + } + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("%s returned %d: %s", endpoint, resp.StatusCode, strings.TrimSpace(string(body))) + } + + return body, nil +} + +// getUser returns the token owner (GET /api/user). +func (c *client) getUser() (user, error) { + var u user + body, err := c.get("/api/user", nil) + if err != nil { + return u, err + } + return u, json.Unmarshal(body, &u) +} + +// listGists pages through a gist-listing endpoint and returns everything. +func (c *client) listGists(path string) ([]gist, error) { + all := []gist{} + for page := 1; ; page++ { + query := url.Values{} + query.Set("page", strconv.Itoa(page)) + query.Set("per_page", strconv.Itoa(perPage)) + + body, err := c.get(path, query) + if err != nil { + return nil, err + } + + batch := []gist{} + if err := json.Unmarshal(body, &batch); err != nil { + return nil, err + } + + all = append(all, batch...) + + if len(batch) < perPage { + break + } + } + return all, nil +} + +// Get lists gists from every configured Opengist source and +// turns them into repositories gickup can clone and back up. +// +// Listing endpoint is chosen per source entry: +// - user set -> GET /api/users/{user}/gists (user's gists) +// - user empty + token -> GET /api/gists (token owner's own gists) +// - user empty + no token -> GET /api/gists/public (public gists only) +// +// Opengist's git-over-HTTP basic auth requires the username to match the token owner, +// so private gists are cloned as that user regardless of which endpoint listed them. +// Public/unlisted gists clone anonymously when no token is set. +func Get(conf *types.Conf) ([]types.Repo, bool) { + ran := false + repos := []types.Repo{} + + for _, repo := range conf.Source.Opengist { + ran = true + + if repo.URL == "" { + log.Error(). + Str("stage", "opengist"). + Msg("no url configured") + continue + } + + base := strings.TrimRight(repo.URL, "/") + sub = logger.CreateSubLogger("stage", "opengist", "url", base) + + token := repo.GetToken() + cl := &client{base: base, token: token, http: &http.Client{Timeout: 30 * time.Second}} + + // The clone username must match the token owner for private gists. + tokenOwner := "" + if token != "" { + u, err := cl.getUser() + if err != nil { + sub.Warn(). + Msgf("couldn't determine token owner, private gists may not clone: %s", err.Error()) + } else { + tokenOwner = u.Username + } + } + + var listPath string + switch { + case repo.User != "": + listPath = "/api/users/" + url.PathEscape(repo.User) + "/gists" + sub.Info(). + Msgf("grabbing gists from %s", repo.User) + case token != "": + listPath = "/api/gists" + sub.Info(). + Msg("grabbing my gists") + default: + listPath = "/api/gists/public" + sub.Info(). + Msg("grabbing public gists") + } + + gists, err := cl.listGists(listPath) + if err != nil { + sub.Error(). + Msg(err.Error()) + continue + } + + include := types.GetMap(repo.Include) + exclude := types.GetMap(repo.Exclude) + + for _, g := range gists { + name := g.SlugURL + if name == "" { + name = g.ID + } + + if exclude[name] { + continue + } + if len(repo.Include) > 0 && !include[name] { + continue + } + + // Origin carries the clone credentials. Opengist wants the token + // owner as the basic-auth username, so override it here. + origin := repo + if tokenOwner != "" { + origin.User = tokenOwner + } + + repos = append(repos, types.Repo{ + Name: name, + URL: g.CloneURL, + SSHURL: g.SSHURL, + Token: token, + Origin: origin, + Owner: g.Owner.Username, + Hoster: types.GetHost(base), + Description: g.Description, + Private: !g.Public, + }) + } + } + + return repos, ran +} diff --git a/opengist/opengist_test.go b/opengist/opengist_test.go new file mode 100644 index 0000000..0df5837 --- /dev/null +++ b/opengist/opengist_test.go @@ -0,0 +1,180 @@ +package opengist + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/cooperspencer/gickup/types" +) + +// gistPayload builds an Opengist-shaped gist JSON object. +func gistPayload(slug, owner string, public bool) map[string]interface{} { + return map[string]interface{}{ + "id": slug, + "slug_url": slug, + "owner": map[string]interface{}{"username": owner}, + "title": slug, + "description": "desc-" + slug, + "public": public, + "visibility": map[bool]string{true: "public", false: "private"}[public], + "clone_url": fmt.Sprintf("http://opengist.example/%s/%s.git", owner, slug), + "ssh_url": fmt.Sprintf("git@opengist.example:%s/%s.git", owner, slug), + } +} + +// newServer returns an httptest server that serves a single page of gists on +// the given path plus a token-owner endpoint, then an empty second page. +func newServer(t *testing.T, path string, gists []map[string]interface{}) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + + mux.HandleFunc("/api/user", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]interface{}{"username": "tokenowner"}) + }) + + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + if page > 1 { + _ = json.NewEncoder(w).Encode([]map[string]interface{}{}) + return + } + _ = json.NewEncoder(w).Encode(gists) + }) + + return httptest.NewServer(mux) +} + +func TestGetOwnGists(t *testing.T) { + t.Parallel() + srv := newServer(t, "/api/gists", []map[string]interface{}{ + gistPayload("alpha", "alice", true), + gistPayload("beta", "alice", false), + }) + defer srv.Close() + + conf := &types.Conf{ + Source: types.Source{ + Opengist: []types.GenRepo{{URL: srv.URL, Token: "og_token"}}, + }, + } + + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + if len(repos) != 2 { + t.Fatalf("expected 2 repos, got %d", len(repos)) + } + + r := repos[0] + if r.Name != "alpha" || r.Owner != "alice" { + t.Fatalf("unexpected repo: %#v", r) + } + if r.URL != "http://opengist.example/alice/alpha.git" { + t.Fatalf("unexpected clone url: %q", r.URL) + } + // Clone auth must use the token owner as username (Opengist requirement). + if r.Origin.User != "tokenowner" { + t.Fatalf("expected clone user tokenowner, got %q", r.Origin.User) + } + if repos[1].Private != true { + t.Fatalf("expected beta to be private") + } +} + +func TestGetPublicGistsAnonymous(t *testing.T) { + t.Parallel() + srv := newServer(t, "/api/gists/public", []map[string]interface{}{ + gistPayload("pub", "bob", true), + }) + defer srv.Close() + + conf := &types.Conf{ + Source: types.Source{ + Opengist: []types.GenRepo{{URL: srv.URL}}, + }, + } + + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + if len(repos) != 1 { + t.Fatalf("expected 1 repo, got %d", len(repos)) + } + if repos[0].Token != "" || repos[0].Origin.User != "" { + t.Fatalf("expected anonymous clone, got token=%q user=%q", repos[0].Token, repos[0].Origin.User) + } +} + +func TestGetUserGists(t *testing.T) { + t.Parallel() + srv := newServer(t, "/api/users/carol/gists", []map[string]interface{}{ + gistPayload("one", "carol", true), + }) + defer srv.Close() + + conf := &types.Conf{ + Source: types.Source{ + Opengist: []types.GenRepo{{URL: srv.URL, Token: "og_token", User: "carol"}}, + }, + } + + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + if len(repos) != 1 || repos[0].Owner != "carol" { + t.Fatalf("unexpected repos: %#v", repos) + } +} + +func TestGetIncludeExclude(t *testing.T) { + t.Parallel() + gists := []map[string]interface{}{ + gistPayload("keep", "alice", true), + gistPayload("drop", "alice", true), + } + + t.Run("include", func(t *testing.T) { + t.Parallel() + srv := newServer(t, "/api/gists", gists) + defer srv.Close() + conf := &types.Conf{Source: types.Source{Opengist: []types.GenRepo{{ + URL: srv.URL, Token: "og_token", Include: []string{"keep"}, + }}}} + repos, _ := Get(conf) + if len(repos) != 1 || repos[0].Name != "keep" { + t.Fatalf("include filter failed: %#v", repos) + } + }) + + t.Run("exclude", func(t *testing.T) { + t.Parallel() + srv := newServer(t, "/api/gists", gists) + defer srv.Close() + conf := &types.Conf{Source: types.Source{Opengist: []types.GenRepo{{ + URL: srv.URL, Token: "og_token", Exclude: []string{"drop"}, + }}}} + repos, _ := Get(conf) + if len(repos) != 1 || repos[0].Name != "keep" { + t.Fatalf("exclude filter failed: %#v", repos) + } + }) +} + +func TestGetSkipsMissingURL(t *testing.T) { + t.Parallel() + conf := &types.Conf{Source: types.Source{Opengist: []types.GenRepo{{}}}} + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + if len(repos) != 0 { + t.Fatalf("expected no repositories, got %d", len(repos)) + } +} diff --git a/radicle/radicle.go b/radicle/radicle.go new file mode 100644 index 0000000..2ba351a --- /dev/null +++ b/radicle/radicle.go @@ -0,0 +1,429 @@ +// Package radicle mirrors repositories into the local storage of a Radicle +// node. +package radicle + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/cooperspencer/gickup/logger" + "github.com/cooperspencer/gickup/types" + "github.com/rs/zerolog" +) + +var sub zerolog.Logger + +// payloadCache memoizes the storage scan so scanStorage inspects the storage +// only once per RAD_HOME, instead of once per mirrored repository. It is +// populated on the first scan and kept in sync as initRepo creates mirrors, +// so this process never re-inspects storage it already knows. +var payloadCache = map[string]map[string]payload{} + +// payload is the identity document `payload` section, as returned by `rad inspect --payload`. +type payload struct { + Project struct { + Name string `json:"name"` + Description string `json:"description"` + DefaultBranch string `json:"defaultBranch"` + } `json:"xyz.radicle.project"` + // Gickup is the provenance extension this package adds to the identity + // document of every mirror it initializes. + Gickup struct { + Source string `json:"source"` + } `json:"dev.gickup"` +} + +// Mirror pushes the repository cloned at tempdir into the Radicle storage +// configured by d, initializing it with `rad init` on first sight. It returns +// the RID of the mirror. +func Mirror(repo types.Repo, dest types.Radicle, tempdir string) (string, error) { + home, _ := Home() + sub = logger.CreateSubLogger("stage", "radicle", "home", home) + + if err := probe(); err != nil { + return "", err + } + + // rad and git-remote-rad read RAD_HOME and RAD_PASSPHRASE from the + // environment themselves, so the ambient environment is all they need. + env := os.Environ() + + nid, err := nodeID(env) + if err != nil { + return "", err + } + + rid, err := scanStorage(repo, nid, env) + if err != nil { + return "", err + } + + if rid == "" { + rid, err = initRepo(repo, dest, tempdir, env) + if err != nil { + return "", err + } + + sub.Info(). + Str("rid", fmt.Sprintf("rad:%s", rid)). + Msgf("initialized %s", types.Green(repo.Name)) + } else if dest.Force || dest.Prune { + if err := fetchMirrorState(tempdir, rid, nid, env); err != nil { + return "", err + } + } + + if err := push(dest, tempdir, rid, nid, env); err != nil { + return "", err + } + + // Issues coming soon + if dest.Issues { + sub.Warn().Msg("Issue migration to Radicle not yet supported.") + } + + return rid, nil +} + +// probe checks that all binaries required for mirroring to Radicle are available. +func probe() error { + for _, bin := range []string{"git", "rad", "git-remote-rad"} { + if _, err := exec.LookPath(bin); err != nil { + return fmt.Errorf("%s is not installed", bin) + } + } + + return nil +} + +// fetchMirrorState fetches the current refs of the mirror into the temporary +// clone. When upstream history was rewritten or a branch was deleted, the old +// ref targets no longer exist in a fresh clone, and the remote helper then +// fails to re-sign rad/sigrefs after the ref update, leaving the signed refs +// pointing at the old state. Fetching first keeps the old objects resolvable +// so refs and signature update together. Pushes without force and prune are +// fast-forward only, where the old targets are always present, so the fetch +// is skipped. +func fetchMirrorState(tempdir, rid, nid string, env []string) error { + _, err := run(tempdir, env, "git", "fetch", "--no-tags", + fmt.Sprintf("rad://%s/%s", rid, nid), + "+refs/heads/*:refs/gickup/heads/*", + "+refs/tags/*:refs/gickup/tags/*") + + return err +} + +// Home returns the profile's Radicle home as reported by `rad self --home`. +// Propagates error accordingly, so it can be handled at call sites. +func Home() (string, error) { + return run("", os.Environ(), "rad", "self", "--home") +} + +// run executes name with args in dir, returning trimmed stdout. Stderr is +// folded into the returned error. +func run(dir string, env []string, name string, args ...string) (string, error) { + cmd := exec.CommandContext(context.Background(), name, args...) + cmd.Dir = dir + cmd.Env = env + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = strings.TrimSpace(stdout.String()) + } + + return "", fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, msg) + } + + return strings.TrimSpace(stdout.String()), nil +} + +// nodeID returns the node ID of the configured profile, which namespaces the +// push URL. `rad self --nid` is deprecated but still works without a running +// node, so it is used as the fallback. +func nodeID(env []string) (string, error) { + nid, err := run("", env, "rad", "node", "status", "--only", "nid") + if err != nil { + nid, err = run("", env, "rad", "self", "--nid") + if err != nil { + return "", err + } + } + + if nid == "" { + return "", fmt.Errorf("couldn't determine the node id of the radicle profile") + } + + return nid, nil +} + +// marker returns the human-readable provenance note that initRepo appends to +// the mirror's description. Any credentials embedded in the URL are stripped, because +// the description becomes part of the signed identity payload that is +// replicated to the network. +func marker(upstream string) string { + return fmt.Sprintf("(mirror of %s)", stripCredentials(upstream)) +} + +// stripCredentials removes the userinfo part from a URL, so tokens or +// passwords configured inline (https://user:token@host/...) never leave the +// local configuration. Non-URL remotes (e.g. scp-style ssh) pass through. This +// is useful because the identity document, where this URL is stored, is signed +// and published to the network. +func stripCredentials(upstream string) string { + if !strings.Contains(upstream, "://") { + return upstream + } + + parsed, err := url.Parse(upstream) + if err != nil || parsed.User == nil { + return upstream + } + + parsed.User = nil + + return parsed.String() +} + +// scanStorage searches the Radicle storage for the repository that mirrors +// the upstream URL. A candidate must record the URL as its source in the +// dev.gickup `payload` of its identity document, and list this profile among +// that identity's delegates. The second check is what makes adoption +// unforgeable: the source claim lives in the identity document, whose contents +// are only authoritative because its delegates signed them. +func scanStorage(repo types.Repo, nid string, env []string) (string, error) { + inspected, err := payloadsFor(env) + if err != nil { + return "", err + } + + matches := []string{} + source := stripCredentials(repo.URL) + + for rid, p := range inspected { + if p.Gickup.Source == source && isDelegate(rid, nid, env) { + matches = append(matches, rid) + } + } + + switch len(matches) { + case 0: + return "", nil + case 1: + return matches[0], nil + default: + return "", fmt.Errorf( + "found %d repositories in storage mirroring %s - remove the duplicates so only one remains", + len(matches), repo.URL) + } +} + +// isDelegate reports whether this profile's key (nid) is a delegate of the +// stored repository's identity. We consider this enough evidence that the +// information we read from the identity doc can be trusted. +func isDelegate(rid, nid string, env []string) bool { + out, err := run("", env, "rad", "inspect", fmt.Sprintf("rad:%s", rid), "--delegates") + if err != nil { + return false + } + + // Each line looks like "did:key: (alias)". + did := "did:key:" + nid + for _, line := range strings.Split(out, "\n") { + if fields := strings.Fields(line); len(fields) > 0 && fields[0] == did { + return true + } + } + + return false +} + +// payloadsFor returns the identity payloads of all repositories in the Radicle +// profile storage. The storage is inspected the first time a given RAD_HOME is asked +// about and the result is cached, so the repeated scanStorage calls across a +// run's repositories don't each re-inspect it. +func payloadsFor(env []string) (map[string]payload, error) { + home, err := Home() + if err != nil { + return nil, err + } + + if cached, ok := payloadCache[home]; ok { + return cached, nil + } + + entries, err := os.ReadDir(filepath.Join(home, "storage")) + if err != nil { + if os.IsNotExist(err) { + payloadCache[home] = map[string]payload{} + return payloadCache[home], nil + } + + return nil, err + } + + current := map[string]payload{} + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + out, err := run("", env, "rad", "inspect", fmt.Sprintf("rad:%s", entry.Name()), "--payload") + if err != nil { + sub.Debug(). + Str("rid", fmt.Sprintf("rad:%s", entry.Name())). + Msg(err.Error()) + + continue + } + + var p payload + if err := json.Unmarshal([]byte(out), &p); err != nil { + continue + } + + current[entry.Name()] = p + } + + payloadCache[home] = current + + return current, nil +} + +// initRepo runs `rad init` on the temporary clone, creating the repository +// identity in storage, and returns the new RID. The upstream URL is recorded +// in a dev.gickup extension of the identity document `payload`, which is what +// scanStorage matches on, on the next run. +func initRepo(repo types.Repo, dest types.Radicle, tempdir string, env []string) (string, error) { + branch, err := run(tempdir, env, "git", "symbolic-ref", "--short", "HEAD") + if err != nil { + return "", err + } + + description := strings.TrimSpace(fmt.Sprintf("%s %s", repo.Description, marker(repo.URL))) + + visibility, err := visibilityFlag(dest, repo) + if err != nil { + return "", err + } + + _, err = run(tempdir, env, "rad", "init", + fmt.Sprintf("--name=%s", repo.Name), + fmt.Sprintf("--description=%s", description), + fmt.Sprintf("--default-branch=%s", branch), + visibility, + "--no-confirm") + if err != nil { + return "", err + } + + rid, err := run(tempdir, env, "rad", ".") + if err != nil { + return "", err + } + + rid = strings.TrimPrefix(rid, "rad:") + + source, err := json.Marshal(stripCredentials(repo.URL)) + if err != nil { + return rid, err + } + + if err := recordMirrorSource(tempdir, env, source, rid); err != nil { + return rid, err + } + + updatePayloadCache(repo, rid) + + return rid, nil +} + +// updatePayloadCache Remember the new mirror so a later run in this process finds it without +// re-inspecting storage — and never tries to initialize it a second time. +func updatePayloadCache(repo types.Repo, rid string) { + home, err := Home() + if err != nil { + return + } + if payloadCache[home] == nil { + payloadCache[home] = map[string]payload{} + } + var newPayload payload + newPayload.Gickup.Source = stripCredentials(repo.URL) + payloadCache[home][rid] = newPayload +} + +// recordMirrorSource Captures the upstream source, so an existing mirror can be reused, +// avoiding duplicates. A failure here fails the overall +// backup, and the error names the repository because the storage offers +// no rollback for a created identity: the operator has to record the +// source or remove the repository by hand. +func recordMirrorSource(tempdir string, env []string, source []byte, rid string) error { + if _, err := run(tempdir, env, "rad", "id", "update", + "--payload", "dev.gickup", "source", string(source), + "--title", "Record mirror source", + "--no-confirm"); err != nil { + return fmt.Errorf( + "repository rad:%s was created but recording its mirror source failed - record dev.gickup.source manually or remove the repository: %w", + rid, err) + } + return nil +} + +// visibilityFlag maps the configured visibility to the rad init flag. +func visibilityFlag(dest types.Radicle, repo types.Repo) (string, error) { + switch dest.Visibility { + case "public": + return "--public", nil + case "private": + return "--private", nil + case "", "source": + if repo.Private { + return "--private", nil + } + + return "--public", nil + default: + return "", fmt.Errorf("invalid radicle visibility %s, must be public, private or source", dest.Visibility) + } +} + +// pushArgs builds the git arguments that mirror all branches and tags into +// the profile's namespace of the stored repository. +func pushArgs(dest types.Radicle, rid, nid string) []string { + prefix := "" + if dest.Force { + prefix = "+" + } + + args := []string{"push"} + if dest.Prune { + args = append(args, "--prune") + } + + return append(args, + fmt.Sprintf("rad://%s/%s", rid, nid), + fmt.Sprintf("%srefs/heads/*:refs/heads/*", prefix), + fmt.Sprintf("%srefs/tags/*:refs/tags/*", prefix)) +} + +// push mirrors all branches and tags of the temporary clone into the profile's +// namespace of the stored repository. The remote helper updates and signs +// rad/sigrefs and announces to the node, so nothing else has to touch storage. +func push(dest types.Radicle, tempdir, rid, nid string, env []string) error { + _, err := run(tempdir, env, "git", pushArgs(dest, rid, nid)...) + + return err +} diff --git a/radicle/radicle_test.go b/radicle/radicle_test.go new file mode 100644 index 0000000..6f1464f --- /dev/null +++ b/radicle/radicle_test.go @@ -0,0 +1,237 @@ +package radicle + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cooperspencer/gickup/types" +) + +func TestHome(t *testing.T) { + // Home delegates to `rad self --home`; stub rad and check we return what + // it reports, verbatim. + bin := t.TempDir() + script := "#!/bin/sh\n[ \"$1 $2\" = \"self --home\" ] && echo /stub/radhome\n" + if err := os.WriteFile(filepath.Join(bin, "rad"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fmt.Sprintf("%s%c%s", bin, os.PathListSeparator, os.Getenv("PATH"))) + + home, err := Home() + if err != nil { + t.Fatalf("Home() error: %v", err) + } + if home != "/stub/radhome" { + t.Errorf("expected /stub/radhome from rad self --home, got %q", home) + } +} + +func TestVisibilityFlag(t *testing.T) { + t.Parallel() + + private := types.Repo{Private: true} + public := types.Repo{Private: false} + + for _, tc := range []struct { + visibility string + repo types.Repo + expected string + }{ + {"public", private, "--public"}, + {"private", public, "--private"}, + {"", private, "--private"}, + {"", public, "--public"}, + {"source", private, "--private"}, + {"source", public, "--public"}, + } { + flag, err := visibilityFlag(types.Radicle{Visibility: tc.visibility}, tc.repo) + if err != nil { + t.Errorf("visibility %q: %v", tc.visibility, err) + } + + if flag != tc.expected { + t.Errorf("visibility %q private=%v: expected %s, got %s", tc.visibility, tc.repo.Private, tc.expected, flag) + } + } + + if _, err := visibilityFlag(types.Radicle{Visibility: "invisible"}, public); err == nil { + t.Error("expected an error for an invalid visibility") + } +} + +func TestPushArgs(t *testing.T) { + t.Parallel() + + args := pushArgs(types.Radicle{}, "zRid", "zNid") + expected := "push rad://zRid/zNid refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" + + if strings.Join(args, " ") != expected { + t.Errorf("expected %q, got %q", expected, strings.Join(args, " ")) + } + + args = pushArgs(types.Radicle{Force: true, Prune: true}, "zRid", "zNid") + expected = "push --prune rad://zRid/zNid +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*" + + if strings.Join(args, " ") != expected { + t.Errorf("expected %q, got %q", expected, strings.Join(args, " ")) + } +} + +const testNID = "zTestNodeId" + +type fakePayload struct { + name string + // source is the upstream URL in the dev.gickup payload; empty simulates + // a repository that was not created by gickup. + source string + // owned repos list this profile among the identity delegates; unowned ones + // list a foreign delegate, simulating a repo merely seeded from the network. + owned bool +} + +// fakeRad puts a rad stub on PATH answering `rad self --home`, `rad inspect +// --payload` and `rad inspect --delegates` per configured rid, and +// creates an (empty) storage directory per rid so payloadsFor can enumerate +// them — letting scanStorage be tested without a radicle installation. +func fakeRad(t *testing.T, radhome string, repos map[string]fakePayload) { + t.Helper() + + bin := t.TempDir() + ourDID := "did:key:" + testNID + + var payloadCases, delegateCases string + for rid, p := range repos { + gickup := "" + if p.source != "" { + gickup = fmt.Sprintf("\"dev.gickup\": {\"source\": \"%s\"}, ", p.source) + } + payloadCases += fmt.Sprintf("rad:%s) echo '{%s\"xyz.radicle.project\": {\"name\": \"%s\", \"defaultBranch\": \"main\"}}' ;;\n", rid, gickup, p.name) + + delegate := "did:key:zForeignNode" + if p.owned { + delegate = ourDID + } + delegateCases += fmt.Sprintf("rad:%s) echo '%s (alias)' ;;\n", rid, delegate) + } + + script := "#!/bin/sh\n" + + "if [ \"$1 $2\" = \"self --home\" ]; then echo \"$RAD_HOME\"; exit 0; fi\n" + + "case \"$3\" in\n" + + "--payload) case \"$2\" in\n" + payloadCases + "*) echo 'unknown repository' >&2; exit 1 ;;\nesac ;;\n" + + "--delegates) case \"$2\" in\n" + delegateCases + "*) echo 'unknown repository' >&2; exit 1 ;;\nesac ;;\n" + + "esac\n" + + if err := os.WriteFile(filepath.Join(bin, "rad"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + + t.Setenv("PATH", fmt.Sprintf("%s%c%s", bin, os.PathListSeparator, os.Getenv("PATH"))) + + for rid := range repos { + if err := os.MkdirAll(filepath.Join(radhome, "storage", rid), 0o755); err != nil { + t.Fatal(err) + } + } +} + +func TestScanStorage(t *testing.T) { + radhome := t.TempDir() + fakeRad(t, radhome, map[string]fakePayload{ + // same repo name upstream, distinguished purely by recorded source + "zRepoOne": {"website", "https://github.com/alice/website", true}, + "zRepoTwo": {"website", "https://gitea.local/bob/website", true}, + "zOther": {"other", "https://github.com/alice/other", true}, + // not created by gickup: no dev.gickup payload + "zForeign": {"website", "", true}, + }) + + t.Setenv("RAD_HOME", radhome) + + rid, err := scanStorage(types.Repo{Name: "website", URL: "https://gitea.local/bob/website"}, testNID, os.Environ()) + if err != nil { + t.Fatalf("scan storage: %v", err) + } + + if rid != "zRepoTwo" { + t.Errorf("expected zRepoTwo, got %s", rid) + } + + rid, err = scanStorage(types.Repo{Name: "website", URL: "https://github.com/carol/website"}, testNID, os.Environ()) + if err != nil { + t.Fatalf("scan storage: %v", err) + } + + if rid != "" { + t.Errorf("expected no match for an unmirrored upstream, got %s", rid) + } + + // credentials in the configured URL don't prevent matching the + // credential-free recorded source + rid, err = scanStorage(types.Repo{Name: "other", URL: "https://user:token@github.com/alice/other"}, testNID, os.Environ()) + if err != nil { + t.Fatalf("scan storage: %v", err) + } + + if rid != "zOther" { + t.Errorf("expected zOther, got %s", rid) + } +} + +func TestScanStorageAmbiguous(t *testing.T) { + radhome := t.TempDir() + url := "https://github.com/alice/same" + fakeRad(t, radhome, map[string]fakePayload{ + "zRepoOne": {"same", url, true}, + "zRepoTwo": {"same", url, true}, + }) + + t.Setenv("RAD_HOME", radhome) + + if _, err := scanStorage(types.Repo{Name: "same", URL: url}, testNID, os.Environ()); err == nil { + t.Error("expected an error for an ambiguous source match") + } +} + +func TestScanStorageRejectsForgedPayloads(t *testing.T) { + radhome := t.TempDir() + victim := "https://github.com/victim/foo" + fakeRad(t, radhome, map[string]fakePayload{ + // a repo seeded from the network whose crafted identity document + // claims the victim's source, but whose delegates don't include this + // profile — the delegate check must reject it + "zSeeded": {"foo", victim, false}, + }) + + t.Setenv("RAD_HOME", radhome) + + rid, err := scanStorage(types.Repo{Name: "foo", URL: victim}, testNID, os.Environ()) + if err != nil { + t.Fatalf("scan storage: %v", err) + } + + if rid != "" { + t.Errorf("forged payload was adopted: %s", rid) + } +} + +func TestMarkerStripsCredentials(t *testing.T) { + t.Parallel() + + m := marker("https://user:secret-token@github.com/alice/repo.git") + + if strings.Contains(m, "secret-token") || strings.Contains(m, "user") { + t.Errorf("marker leaks credentials: %s", m) + } + + if m != "(mirror of https://github.com/alice/repo.git)" { + t.Errorf("unexpected marker: %s", m) + } + + // non-URL remotes pass through untouched + if marker("git@github.com:alice/repo.git") != "(mirror of git@github.com:alice/repo.git)" { + t.Error("scp-style remote was altered") + } +} diff --git a/s3/s3.go b/s3/s3.go index 73689fd..b2ae944 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -14,11 +14,19 @@ import ( var sub zerolog.Logger +// getCredentials returns the appropriate credentials based on the S3Repo configuration +func getCredentials(s3repo types.S3Repo) *credentials.Credentials { + if s3repo.UseStaticCreds { + return credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, s3repo.Token) + } + return credentials.NewIAM("") +} + // UploadDirToS3 uploads the contents of a directory to S3-compatible storage func UploadDirToS3(directory string, s3repo types.S3Repo, options *minio.PutObjectOptions) error { // Initialize minio client object. client, err := minio.New(s3repo.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, s3repo.Token), + Creds: getCredentials(s3repo), Secure: s3repo.UseSSL, Region: s3repo.Region, }) @@ -71,7 +79,7 @@ func DeleteObjectsNotInRepo(directory, bucketdir string, s3repo types.S3Repo) er sub = logger.CreateSubLogger("stage", "s3", "endpoint", s3repo.Endpoint, "bucket", s3repo.Bucket) // Initialize minio client object. client, err := minio.New(s3repo.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, s3repo.Token), + Creds: getCredentials(s3repo), Secure: s3repo.UseSSL, Region: s3repo.Region, }) diff --git a/s3/s3_test.go b/s3/s3_test.go new file mode 100644 index 0000000..d52b5a4 --- /dev/null +++ b/s3/s3_test.go @@ -0,0 +1,49 @@ +package s3 + +import ( + "testing" + + "github.com/cooperspencer/gickup/types" +) + +func TestGetCredentialsStaticCredsReturnsStaticValues(t *testing.T) { + t.Parallel() + + s3repo := types.S3Repo{ + UseStaticCreds: true, + AccessKey: "test-access", + SecretKey: "test-secret", + Token: "test-token", + } + + creds := getCredentials(s3repo) + if creds == nil { + t.Fatal("expected non-nil credentials") + } + + val, err := creds.Get() + if err != nil { + t.Fatalf("Get() error = %v", err) + } + + if val.AccessKeyID != "test-access" { + t.Fatalf("AccessKeyID = %q, want %q", val.AccessKeyID, "test-access") + } + + if val.SecretAccessKey != "test-secret" { + t.Fatalf("SecretAccessKey = %q, want %q", val.SecretAccessKey, "test-secret") + } + + if val.SessionToken != "test-token" { + t.Fatalf("SessionToken = %q, want %q", val.SessionToken, "test-token") + } +} + +func TestGetCredentialsIAMReturnsNonNil(t *testing.T) { + t.Parallel() + + creds := getCredentials(types.S3Repo{UseStaticCreds: false}) + if creds == nil { + t.Fatal("expected non-nil IAM credentials") + } +} diff --git a/sourcehut/sourcehut.go b/sourcehut/sourcehut.go index ac0ef07..1ceb4ca 100644 --- a/sourcehut/sourcehut.go +++ b/sourcehut/sourcehut.go @@ -13,11 +13,8 @@ import ( "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" graphqlclient "github.com/hasura/go-graphql-client" - "github.com/rs/zerolog" ) -var sub zerolog.Logger - const defaultSourcehutURL = "https://git.sr.ht" func normalizeBearerToken(token string) string { @@ -250,7 +247,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { for _, repo := range conf.Source.Sourcehut { repo.URL = normalizeURL(repo.URL) - sub = logger.CreateSubLogger("stage", "sourcehut", "url", repo.URL) + sub := logger.CreateSubLogger("stage", "sourcehut", "url", repo.URL) err := repo.Filter.ParseDuration() if err != nil { sub.Warn(). @@ -306,6 +303,10 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { ownerName := strings.TrimPrefix(ownerCanonicalName, "~") isPrivate := strings.EqualFold(r.Visibility, "PRIVATE") || strings.EqualFold(r.Visibility, "private") + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -335,10 +336,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, @@ -374,8 +371,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { destination.URL = normalizeURL(destination.URL) - sub = logger.CreateSubLogger("stage", "sourcehut", "url", destination.URL) - token := destination.GetToken() endpoint := graphQLEndpoint(destination.URL) configuredUser := strings.TrimPrefix(destination.User, "~") diff --git a/types/types.go b/types/types.go index 31ad9c9..1ef9443 100644 --- a/types/types.go +++ b/types/types.go @@ -22,15 +22,17 @@ import ( // Destination TODO. type Destination struct { - Gitlab []GenRepo `yaml:"gitlab"` - Local []Local `yaml:"local"` - Github []GenRepo `yaml:"github"` - Gitea []GenRepo `yaml:"gitea"` - Gogs []GenRepo `yaml:"gogs"` - OneDev []GenRepo `yaml:"onedev"` - Sourcehut []GenRepo `yaml:"sourcehut"` - S3 []S3Repo `yaml:"s3"` - AzureBlob []AzureBlob `yaml:"azureblob"` + Gitlab []GenRepo `yaml:"gitlab"` + Local []Local `yaml:"local"` + Github []GenRepo `yaml:"github"` + Gitea []GenRepo `yaml:"gitea"` + Gogs []GenRepo `yaml:"gogs"` + OneDev []GenRepo `yaml:"onedev"` + Sourcehut []GenRepo `yaml:"sourcehut"` + S3 []S3Repo `yaml:"s3"` + AzureBlob []AzureBlob `yaml:"azureblob"` + WebDAV []WebDAVRepo `yaml:"webdav"` + Radicle []Radicle `yaml:"radicle"` } // Count TODO. @@ -43,7 +45,21 @@ func (dest Destination) Count() int { len(dest.OneDev) + len(dest.Sourcehut) + len(dest.S3) + - len(dest.AzureBlob) + len(dest.AzureBlob) + + len(dest.WebDAV) + + len(dest.Radicle) +} + +// Radicle : The configuration for mirroring to Radicle. +// Repositories are initialized with `rad init` and pushed into the local +// Radicle storage through the git-remote-rad helper, signed with the key of +// the profile in RAD_HOME. When that key has a passphrase, export it in the +// RAD_PASSPHRASE environment variable, which rad reads itself. +type Radicle struct { + Force bool `yaml:"force"` // overwrite diverged refs on the mirror (non-fast-forward updates) + Prune bool `yaml:"prune"` // delete refs on the mirror that no longer exist upstream + Visibility string `yaml:"visibility"` // public, private or source, default: source + Issues bool `yaml:"issues"` // [NOT YET SUPPORTED] recreate upstream issues as radicle issues (source must also fetch them with issues: true) } // Local TODO. @@ -57,6 +73,11 @@ type Local struct { LFS bool `yaml:"lfs"` } +// WebUIConfig holds web interface settings. +type WebUIConfig struct { + Addr string `yaml:"addr"` +} + // Conf TODO. type Conf struct { Source Source `yaml:"source"` @@ -64,6 +85,7 @@ type Conf struct { Cron string `yaml:"cron"` Log Logging `yaml:"log"` Metrics Metrics `yaml:"metrics"` + WebUI WebUIConfig `yaml:"webui"` } // PrometheusConfig TODO. @@ -74,7 +96,8 @@ type PrometheusConfig struct { // HeartbeatConfig TODO. type HeartbeatConfig struct { - URLs []string `yaml:"urls"` + URLs []string `yaml:"urls"` + FailureURLs []string `yaml:"failure_urls"` } // PushConfig TODO. @@ -229,6 +252,7 @@ type Source struct { BitBucket []GenRepo `yaml:"bitbucket"` OneDev []GenRepo `yaml:"onedev"` Sourcehut []GenRepo `yaml:"sourcehut"` + Opengist []GenRepo `yaml:"opengist"` Any []GenRepo `yaml:"any"` } @@ -241,36 +265,41 @@ func (source Source) Count() int { len(source.Gitlab) + len(source.OneDev) + len(source.Sourcehut) + + len(source.Opengist) + len(source.Any) } // GenRepo Generell Repo. type GenRepo struct { - Token string `yaml:"token"` - TokenFile string `yaml:"token_file"` - User string `yaml:"user"` - Organization string `yaml:"organization"` - SSH bool `yaml:"ssh"` - SSHKey string `yaml:"sshkey"` - Username string `yaml:"username"` - Password string `yaml:"password"` - URL string `yaml:"url"` - Exclude []string `yaml:"exclude"` - ExcludeOrgs []string `yaml:"excludeorgs"` - Include []string `yaml:"include"` - IncludeOrgs []string `yaml:"includeorgs"` - Issues bool `yaml:"issues"` - Wiki bool `yaml:"wiki"` - Starred bool `yaml:"starred"` - CreateOrg bool `yaml:"createorg"` - Visibility Visibility `yaml:"visibility"` - Filter Filter `yaml:"filter"` - Force bool `yaml:"force"` - Contributed bool `yaml:"contributed"` - MirrorInterval string `yaml:"mirrorinterval"` - LFS bool `yaml:"lfs"` - Mirror Mirror `yaml:"mirror"` - Gists bool `yaml:"gists"` + Token string `yaml:"token"` + TokenFile string `yaml:"token_file"` + User string `yaml:"user"` + Email string `yaml:"email"` + Organization string `yaml:"organization"` + SSH bool `yaml:"ssh"` + SSHKey string `yaml:"sshkey"` + Username string `yaml:"username"` + Password string `yaml:"password"` + URL string `yaml:"url"` + Exclude []string `yaml:"exclude"` + ExcludeOrgs []string `yaml:"excludeorgs"` + Include []string `yaml:"include"` + IncludeOrgs []string `yaml:"includeorgs"` + Issues bool `yaml:"issues"` + Wiki bool `yaml:"wiki"` + Starred bool `yaml:"starred"` + CreateOrg bool `yaml:"createorg"` + Visibility Visibility `yaml:"visibility"` + Filter Filter `yaml:"filter"` + Force bool `yaml:"force"` + Contributed bool `yaml:"contributed"` + MirrorInterval string `yaml:"mirrorinterval"` + LFS bool `yaml:"lfs"` + Mirror Mirror `yaml:"mirror"` + Gists bool `yaml:"gists"` + AppID int64 `yaml:"app_id"` + AppInstallationID int64 `yaml:"app_installation_id"` + AppPrivateKeyFile string `yaml:"app_private_key_file"` } // Mirror struct @@ -308,6 +337,11 @@ func (grepo GenRepo) GetToken() string { return token } +// HasAppAuth returns true when all GitHub App authentication fields are set. +func (grepo GenRepo) HasAppAuth() bool { + return grepo.AppID > 0 && grepo.AppInstallationID > 0 && grepo.AppPrivateKeyFile != "" +} + func (f *Filter) ParseDuration() error { rest := strings.Trim(f.LastActivityString, " ") date := time.Now() @@ -429,20 +463,24 @@ type GithubDestination struct { } // GetHost TODO. -func GetHost(url string) string { - if strings.Contains(url, "http://") { - url = strings.Split(url, "http://")[1] +func GetHost(rawURL string) string { + if endpoint, err := transport.NewEndpoint(rawURL); err == nil && endpoint.Host != "" { + return endpoint.Host + } + + if strings.Contains(rawURL, "http://") { + rawURL = strings.Split(rawURL, "http://")[1] } - if strings.Contains(url, "https://") { - url = strings.Split(url, "https://")[1] + if strings.Contains(rawURL, "https://") { + rawURL = strings.Split(rawURL, "https://")[1] } - if strings.Contains(url, "/") { - url = strings.Split(url, "/")[0] + if strings.Contains(rawURL, "/") { + rawURL = strings.Split(rawURL, "/")[0] } - return url + return rawURL } // GetValues TODO. @@ -552,6 +590,7 @@ func StatRemote(remoteURL, sshURL string, repo GenRepo) bool { type S3Repo struct { Bucket string `yaml:"bucket"` Endpoint string `yaml:"endpoint"` + UseStaticCreds bool `yaml:"use_static_creds" default:"true"` AccessKey string `yaml:"accesskey"` SecretKey string `yaml:"secretkey"` Token string `yaml:"token"` @@ -562,6 +601,7 @@ type S3Repo struct { StorageClass string `yaml:"storageclass"` DateCreateDir bool `yaml:"datecreatedir"` SrcRepoUrlTagKey *string `yaml:"src_repo_url_tag_key"` + LFS bool `yaml:"lfs"` } func (s3 S3Repo) GetKey(accessString string) (string, error) { @@ -590,3 +630,13 @@ type AzureBlob struct { Zip bool `yaml:"zip"` DateCreateDir bool `yaml:"datecreatedir"` } + +type WebDAVRepo struct { + Url string `yaml:"url"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Path string `yaml:"path"` + Structured bool `yaml:"structured"` + Zip bool `yaml:"zip"` + DateCreateDir bool `yaml:"datecreatedir"` +} diff --git a/types/types_test.go b/types/types_test.go index d70f7a0..6195145 100644 --- a/types/types_test.go +++ b/types/types_test.go @@ -223,9 +223,12 @@ func TestGetHost(t *testing.T) { t.Parallel() tests := map[string]string{ - "https://example.com/org/repo": "example.com", - "http://example.com/org/repo": "example.com", - "example.com/org/repo": "example.com", + "https://example.com/org/repo": "example.com", + "http://example.com/org/repo": "example.com", + "ssh://alice%40example.com@git.example.com/org/repo.git": "git.example.com", + "ssh://git@example.com:2222/org/repo.git": "example.com", + "git@example.com:org/repo.git": "example.com", + "example.com/org/repo": "example.com", } for input, want := range tests { @@ -249,6 +252,70 @@ func TestSiteGetValuesSSHURL(t *testing.T) { } } +func TestHasAppAuthReturnsTrueWhenAllFieldsSet(t *testing.T) { + t.Parallel() + + repo := GenRepo{ + AppID: 12345, + AppInstallationID: 67890, + AppPrivateKeyFile: "/path/to/key.pem", + } + + if !repo.HasAppAuth() { + t.Fatal("expected HasAppAuth() = true when all fields are set") + } +} + +func TestHasAppAuthReturnsFalseWhenAppIDMissing(t *testing.T) { + t.Parallel() + + repo := GenRepo{ + AppID: 0, + AppInstallationID: 67890, + AppPrivateKeyFile: "/path/to/key.pem", + } + + if repo.HasAppAuth() { + t.Fatal("expected HasAppAuth() = false when AppID is zero") + } +} + +func TestHasAppAuthReturnsFalseWhenInstallationIDMissing(t *testing.T) { + t.Parallel() + + repo := GenRepo{ + AppID: 12345, + AppInstallationID: 0, + AppPrivateKeyFile: "/path/to/key.pem", + } + + if repo.HasAppAuth() { + t.Fatal("expected HasAppAuth() = false when AppInstallationID is zero") + } +} + +func TestHasAppAuthReturnsFalseWhenPrivateKeyFileMissing(t *testing.T) { + t.Parallel() + + repo := GenRepo{ + AppID: 12345, + AppInstallationID: 67890, + AppPrivateKeyFile: "", + } + + if repo.HasAppAuth() { + t.Fatal("expected HasAppAuth() = false when AppPrivateKeyFile is empty") + } +} + +func TestHasAppAuthReturnsFalseForEmptyRepo(t *testing.T) { + t.Parallel() + + if (GenRepo{}).HasAppAuth() { + t.Fatal("expected HasAppAuth() = false for zero-value GenRepo") + } +} + func TestSiteGetValuesScpStyle(t *testing.T) { t.Parallel() @@ -308,3 +375,21 @@ func TestResolveTrimsNothingFromLiteral(t *testing.T) { t.Fatalf("resolve() = %q, want already-set", got) } } + +func TestS3RepoUseStaticCredsDefaultFalse(t *testing.T) { + t.Parallel() + + var s3 S3Repo + if s3.UseStaticCreds { + t.Fatal("expected UseStaticCreds to be false by default (Go zero value)") + } +} + +func TestS3RepoUseStaticCredsExplicitTrue(t *testing.T) { + t.Parallel() + + s3 := S3Repo{UseStaticCreds: true} + if !s3.UseStaticCreds { + t.Fatal("expected UseStaticCreds to be true when explicitly set") + } +} diff --git a/webdav/webdav.go b/webdav/webdav.go new file mode 100644 index 0000000..fe7c189 --- /dev/null +++ b/webdav/webdav.go @@ -0,0 +1,207 @@ +package webdav + +import ( + "context" + "errors" + "fmt" + "io" + "math/rand/v2" + "net" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/cooperspencer/gickup/logger" + "github.com/cooperspencer/gickup/types" + "github.com/rs/zerolog" + gowebdav "github.com/studio-b12/gowebdav" +) + +var sub zerolog.Logger + +func newClient(repo types.WebDAVRepo) *gowebdav.Client { + t, ok := http.DefaultTransport.(*http.Transport) + if !ok { + t = &http.Transport{} + } + t = t.Clone() + t.ResponseHeaderTimeout = 60 * time.Second + c := gowebdav.NewClient(repo.Url, repo.Username, repo.Password) + c.SetTransport(&retryRoundTripper{base: t}) + return c +} + +// UploadDirToWebDAV uploads the contents of a directory to a WebDAV server +func UploadDirToWebDAV(directory string, repo types.WebDAVRepo) error { + sub = logger.CreateSubLogger("stage", "webdav", "url", repo.Url) + c := newClient(repo) + + return filepath.Walk(directory, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil + } + rel := filepath.ToSlash(p[len(directory)+1:]) + remote := path.Join(repo.Path, rel) + + f, err := os.Open(p) + if err != nil { + return err + } + defer f.Close() + return c.WriteStreamWithLength(remote, f, info.Size(), 0o644) + }) +} + +// DeleteObjectsNotInRepo deletes files from the WebDAV server that are not present in the repository +func DeleteObjectsNotInRepo(directory, repoName string, repo types.WebDAVRepo) error { + sub = logger.CreateSubLogger("stage", "webdav", "url", repo.Url) + c := newClient(repo) + root := path.Join(repo.Path, repoName) + + keys, err := walkRemote(c, root) + if err != nil { + // Nothing uploaded yet. + if gowebdav.IsErrNotFound(err) { + return nil + } + return err + } + for _, key := range keys { + if _, statErr := os.Stat(filepath.Join(directory, repoName, filepath.FromSlash(key))); statErr == nil { + continue + } else if !os.IsNotExist(statErr) { + return statErr + } + sub.Debug().Msgf("Removing %s from webdav", key) + if err := c.RemoveAll(path.Join(root, key)); err != nil { + return err + } + } + + return nil +} + +// walkRemote returns the keys relative to dir of every regular file beneath it. +func walkRemote(c *gowebdav.Client, dir string) ([]string, error) { + var files []string + prefix := strings.TrimSuffix(dir, "/") + "/" + visited := map[string]bool{} + + var walk func(current string) error + walk = func(current string) error { + if visited[current] { + return nil + } + visited[current] = true + entries, err := c.ReadDir(current) + if err != nil { + return err + } + for _, e := range entries { + child := path.Join(current, e.Name()) + if e.IsDir() { + if err := walk(child); err != nil { + return err + } + continue + } + files = append(files, strings.TrimPrefix(child, prefix)) + } + return nil + } + if err := walk(dir); err != nil { + return nil, err + } + return files, nil +} + +// retryRoundTripper retries transient failures with exponential backoff, +// rewinding seekable bodies via GetBody so PUTs replay the full payload. +type retryRoundTripper struct { + base http.RoundTripper +} + +func (r *retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + const maxAttempts = 3 + backoff := 500 * time.Millisecond + + resp, err := r.base.RoundTrip(req) + for attempt := 1; shouldRetry(resp, err) && attempt < maxAttempts; attempt++ { + // Drain and close the failed response so its connection can be reused. + if resp != nil { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 512)) + resp.Body.Close() + } + + // Rewind seekable bodies via GetBody so retried PUTs replay the full payload. + if req.GetBody != nil { + body, berr := req.GetBody() + if berr != nil { + return nil, fmt.Errorf("retry %s %s: rewind body: %w", req.Method, req.URL.Redacted(), berr) + } + req.Body = body + } + + wait := backoff + time.Duration(rand.Int64N(int64(backoff/2))) + reason := "unknown" + switch { + case err != nil: + reason = err.Error() + case resp != nil: + reason = resp.Status + } + sub.Warn().Msgf("%s %s: attempt %d failed (%s); retrying in %v", + req.Method, req.URL.Redacted(), attempt, reason, wait) + + // Back off with jitter, bailing out immediately if the request was cancelled. + timer := time.NewTimer(wait) + select { + case <-timer.C: + case <-req.Context().Done(): + timer.Stop() + return resp, req.Context().Err() + } + backoff *= 2 + + resp, err = r.base.RoundTrip(req) + } + return resp, err +} + +func shouldRetry(resp *http.Response, err error) bool { + if err != nil { + // Context cancellations are intentional, not transient + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + // Only retry transient network failures + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return true + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + if errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNREFUSED) || + errors.Is(err, syscall.EPIPE) { + return true + } + return false + } + switch resp.StatusCode { + case http.StatusTooManyRequests, // 429 + http.StatusInternalServerError, // 500 + http.StatusBadGateway, // 502 + http.StatusServiceUnavailable, // 503 + http.StatusGatewayTimeout: // 504 + return true + } + return false +} diff --git a/webdav/webdav_test.go b/webdav/webdav_test.go new file mode 100644 index 0000000..f93d41b --- /dev/null +++ b/webdav/webdav_test.go @@ -0,0 +1,88 @@ +package webdav + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "syscall" + "testing" + "time" +) + +// fakeNetError is a minimal net.Error for table tests. +type fakeNetError struct{ timeout bool } + +func (fakeNetError) Error() string { return "net error" } +func (e fakeNetError) Timeout() bool { return e.timeout } +func (fakeNetError) Temporary() bool { return false } + +func TestShouldRetry(t *testing.T) { + t.Parallel() + cases := []struct { + name string + resp *http.Response + err error + want bool + }{ + {"503", &http.Response{StatusCode: http.StatusServiceUnavailable}, nil, true}, + {"429", &http.Response{StatusCode: http.StatusTooManyRequests}, nil, true}, + {"500", &http.Response{StatusCode: http.StatusInternalServerError}, nil, true}, + {"502", &http.Response{StatusCode: http.StatusBadGateway}, nil, true}, + {"504", &http.Response{StatusCode: http.StatusGatewayTimeout}, nil, true}, + {"200", &http.Response{StatusCode: http.StatusOK}, nil, false}, + {"404", &http.Response{StatusCode: http.StatusNotFound}, nil, false}, + {"401", &http.Response{StatusCode: http.StatusUnauthorized}, nil, false}, + {"net timeout", nil, fakeNetError{timeout: true}, true}, + {"eof", nil, io.EOF, true}, + {"unexpected eof", nil, io.ErrUnexpectedEOF, true}, + {"conn reset", nil, syscall.ECONNRESET, true}, + {"conn refused", nil, syscall.ECONNREFUSED, true}, + {"broken pipe", nil, syscall.EPIPE, true}, + {"wrapped eof", nil, fmt.Errorf("wrap: %w", io.EOF), true}, + {"context canceled", nil, context.Canceled, false}, + {"context deadline", nil, context.DeadlineExceeded, false}, + {"plain error", nil, errors.New("boom"), false}, + {"non-timeout net", nil, fakeNetError{timeout: false}, false}, + } + for _, c := range cases { + if got := shouldRetry(c.resp, c.err); got != c.want { + t.Errorf("%s: got %v want %v", c.name, got, c.want) + } + } +} + +// TestRetryAbortsOnContextCancel verifies the backoff sleep yields to a +// canceled request context instead of blocking for the full retry window. +func TestRetryAbortsOnContextCancel(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(60*time.Millisecond, cancel) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL, nil) + if err != nil { + t.Fatal(err) + } + + start := time.Now() + resp, err := (&retryRoundTripper{base: http.DefaultTransport}).RoundTrip(req) + elapsed := time.Since(start) + if resp != nil { + resp.Body.Close() + } + + if err == nil { + t.Fatal("expected error from canceled context") + } + if elapsed > 400*time.Millisecond { + t.Fatalf("retry did not abort on cancel: %v (backoff base is 500ms)", elapsed) + } +} diff --git a/webui/ui/index.html b/webui/ui/index.html new file mode 100644 index 0000000..4fb4112 --- /dev/null +++ b/webui/ui/index.html @@ -0,0 +1,1076 @@ + + + + + + Gickup + + + + + + + +
+ + + +
+
+
Total Backups
—
+
Successful
—
+
Failed
—
+
Unique Repos
—
+
Source Hosts
—
+
+ +
+
+

Run Backup

+
+ +
+
+
+ + +
+ +
+ +
+
+

Backup History

+
+ + + +
+
+ +

Counts and filters reflect recorded backups since startup or the last history clear.

+
+ + + + +
+ +
+
+
📦
+
No backup data yet
+
Backup entries will appear here once a run completes.
+
+
+
+
+ + +
+
+
+

Configuration

+
+ +
+
+
+
+

+
+ Configuration example and supported providers +

Sources: GitHub, GitLab, Gitea/Forgejo, Gogs, Bitbucket, OneDev, SourceHut, Opengist, or any Git URL.

+

Destinations: local, GitHub, GitLab, Gitea/Forgejo, Gogs, OneDev, SourceHut, S3, Azure Blob, WebDAV, or Radicle.

+

Copy this example into the editor and replace the repository URL and backup path. Without a cron schedule, saving starts a backup after the configuration is loaded.

+
source:
+  any:
+    - url: https://github.com/OWNER/REPOSITORY.git
+destination:
+  local:
+    - path: /backups
+      bare: true
+# Optional daily schedule (UTC unless TZ is set):
+# cron: "0 0 * * *"
+
+

Edit configuration as YAML. Reload reads the file from disk and discards edits. Saved changes are picked up automatically; without a schedule, reloading the configuration starts a backup.

+ +
+ + +
+
+
+
+ + +
+
+

API Reference

+
+ +
+
GET/api/status
+
Return all recorded backup entries as a JSON array.
+
+
+
DELETE/api/status
+
Clear all recorded backup entries.
+
+ +
+
GET/api/configs
+
Return the list of loaded config blocks (name, sources, destinations, cron spec, next run).
+
+ +
+
GET/api/running
+
Returns {"running": true|false} — whether a backup run is currently in progress.
+
+ +
+
POST/api/run
+
Trigger an immediate backup run.
+ Body: {"index": -1} — -1 runs all configs, ≥0 runs the config at that index.
+ Returns 202 Accepted or 409 Conflict if already running. +
+
+ +
+
GET/api/config
+
Read a config file.
+ ?file=0 — zero-based file index (default 0).
+ ?list=1 — list all registered config files as JSON instead. +
+
+
+
POST/api/config
+
Overwrite a config file with the plain-text request body.
+ ?file=0 — zero-based file index (default 0). +
+
+ +
+
+
+ +
+ + + + diff --git a/webui/ui/index.test.cjs b/webui/ui/index.test.cjs new file mode 100644 index 0000000..ea2ad54 --- /dev/null +++ b/webui/ui/index.test.cjs @@ -0,0 +1,94 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const vm = require('node:vm'); + +function setup() { + const elements = new Map(); + const element = id => { + if (!elements.has(id)) elements.set(id, { + value: '', innerHTML: '', textContent: '', style: {}, disabled: false, + classList: { add() {}, remove() {}, toggle() {} }, + querySelectorAll: () => [], + }); + return elements.get(id); + }; + const listeners = {}; + const context = vm.createContext({ + document: { + getElementById: element, querySelectorAll: () => [], + documentElement: { setAttribute() {} }, + }, + localStorage: { getItem: () => 'dark', setItem() {} }, + window: { matchMedia: () => ({ matches: true }), addEventListener: (name, fn) => listeners[name] = fn }, + confirm: () => false, console, setInterval: () => 1, + fetch: async () => { throw new Error('Unexpected request'); }, + }); + const html = readFileSync(__dirname + '/index.html', 'utf8'); + const script = html.match(/