From cbe54b773d5bcfc05b2543c9a5491e09e75299cc Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Sat, 2 May 2026 21:11:22 +0200 Subject: [PATCH 01/44] added webinterface --- conf.example.yml | 3 + gickup_spec.json | 11 + main.go | 168 +++++++- types/types.go | 6 + webui/ui/index.html | 976 ++++++++++++++++++++++++++++++++++++++++++++ webui/ui/logo.png | Bin 0 -> 35084 bytes webui/webui.go | 282 +++++++++++++ 7 files changed, 1444 insertions(+), 2 deletions(-) create mode 100644 webui/ui/index.html create mode 100644 webui/ui/logo.png create mode 100644 webui/webui.go diff --git a/conf.example.yml b/conf.example.yml index 274ec366..ed5cd876 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -293,6 +293,9 @@ destination: structured: true # checks repos out like hostersite/user|organization/repo zip: false # if true, will zip the entire git repo into a single zip file and upload that instead datecreatedir: false # if true, gickup will create a directory for backup with the creation date +webui: # optional - enables the built-in web interface + addr: ":8282" # address and port to listen on + cron: 0 22 * * * # optional - when cron is not provided, the program runs once and exits. # Otherwise, it runs according to the cron schedule. # See timezone commentary in docker-compose.yml for making sure this container runs diff --git a/gickup_spec.json b/gickup_spec.json index 41e8c1c2..e438fb23 100644 --- a/gickup_spec.json +++ b/gickup_spec.json @@ -15,6 +15,17 @@ "metrics": { "$ref": "#/definitions/metrics" }, + "webui": { + "type": "object", + "description": "Configuration for the built-in web interface", + "additionalProperties": false, + "properties": { + "addr": { + "type": "string", + "description": "Address and port to listen on, e.g. ':8282'" + } + } + }, "source": { "type": "object", "description": "Configuration for all git hoster source endpoints", diff --git a/main.go b/main.go index fd354267..30fa721e 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "reflect" "strconv" "strings" + "sync" "time" "github.com/alecthomas/kong" @@ -31,6 +32,7 @@ import ( "github.com/cooperspencer/gickup/s3" "github.com/cooperspencer/gickup/sourcehut" "github.com/cooperspencer/gickup/types" + "github.com/cooperspencer/gickup/webui" "github.com/cooperspencer/gickup/whatever" "github.com/cooperspencer/gickup/zip" "github.com/go-git/go-git/v5" @@ -185,6 +187,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "local", d.Path).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("local").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "local", + DestAddr: d.Path, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } for _, d := range conf.Destination.S3 { @@ -296,6 +309,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "s3", d.Endpoint).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("s3").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "s3", + DestAddr: d.Endpoint, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -387,6 +411,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "azureblob", d.Container).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("azureblob").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "azureblob", + DestAddr: d.Container, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -475,6 +510,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "gitea", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -557,6 +603,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gogs").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "gogs", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -643,6 +700,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitlab", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gitlab").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "gitlab", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -718,6 +786,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "github", "https://github.com").Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("github").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "github", + DestAddr: "https://github.com", + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } } @@ -796,6 +875,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "onedev", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("onedev").Inc() os.RemoveAll(tempdir) + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "onedev", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } } @@ -875,6 +965,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "sourcehut", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("sourcehut").Inc() os.RemoveAll(tempdir) + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "sourcehut", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } } @@ -1014,8 +1115,10 @@ func playsForever(c *cron.Cron, conffiles []string, confs []*types.Conf) bool { if !cmp.Equal(confs, checkconfigs) { log.Info().Msg("config changed") log.Debug().Msg(cmp.Diff(confs, checkconfigs)) - for _, entry := range c.Entries() { - c.Remove(entry.ID) + if c != nil { + for _, entry := range c.Entries() { + c.Remove(entry.ID) + } } return true } @@ -1064,10 +1167,15 @@ func main() { Msgf("this is a %s", types.Blue("dry run")) } + var webuiOnce sync.Once + var webuiAddr string + _ = webuiAddr // will be set from config + init := true for { reload := false confs := []*types.Conf{} + confNames := []string{} for i, f := range cli.Configfiles { log.Info().Str("file", f). Msgf("Reading %s", types.Green(f)) @@ -1076,7 +1184,17 @@ func main() { log.Panic().Err(err).Msgf("there is an issue with %s", f) } cli.Configfiles[i] = absf + base := filepath.Base(absf) + before := len(confs) confs = append(confs, readConfigFile(absf)...) + added := len(confs) - before + for j := 0; j < added; j++ { + if added > 1 { + confNames = append(confNames, fmt.Sprintf("%s #%d", base, j+1)) + } else { + confNames = append(confNames, base) + } + } } logConf := confs[0].Log @@ -1087,6 +1205,46 @@ func main() { log.Logger = logger.CreateLogger(logConf) + // Determine webUI address from config file. + if webuiAddr == "" && confs[0].WebUI.Addr != "" { + webuiAddr = confs[0].WebUI.Addr + } + + // Register config summaries and run callback; start server once. + if webuiAddr != "" { + infos := make([]webui.ConfigInfo, len(confs)) + for i, c := range confs { + info := webui.ConfigInfo{ + Index: i, + Name: confNames[i], + Sources: c.Source.Count(), + Dests: c.Destination.Count(), + } + if c.HasValidCronSpec() { + info.CronSpec = c.Cron + if next, err := c.GetNextRun(); err == nil { + info.NextRun = next.Format(time.RFC3339) + } + } + infos[i] = info + } + webui.Global.SetConfigFiles(cli.Configfiles) + webui.Global.SetConfigs(infos) + currentConfs := confs + webui.Global.SetRunFunc(func(idx int) { + if idx < 0 { + for i, c := range currentConfs { + runBackup(c, i) + } + } else if idx < len(currentConfs) { + runBackup(currentConfs[idx], idx) + } + }) + webuiOnce.Do(func() { + go webui.Serve(webuiAddr) + }) + } + validcron := confs[0].HasValidCronSpec() var c *cron.Cron @@ -1145,6 +1303,12 @@ func main() { } reload = playsForever(c, cli.Configfiles, confs) log.Info().Msg("reloading config...") + } else if webuiAddr != "" { + // No cron but web UI is active: stay alive and watch for config changes. + reload = playsForever(nil, cli.Configfiles, confs) + if reload { + log.Info().Msg("reloading config...") + } } if !reload { break diff --git a/types/types.go b/types/types.go index 31ad9c91..c81780d7 100644 --- a/types/types.go +++ b/types/types.go @@ -57,6 +57,11 @@ type Local struct { LFS bool `yaml:"lfs"` } +// WebUIConfig holds web interface settings. +type WebUIConfig struct { + Addr string `yaml:"addr"` +} + // Conf TODO. type Conf struct { Source Source `yaml:"source"` @@ -64,6 +69,7 @@ type Conf struct { Cron string `yaml:"cron"` Log Logging `yaml:"log"` Metrics Metrics `yaml:"metrics"` + WebUI WebUIConfig `yaml:"webui"` } // PrometheusConfig TODO. diff --git a/webui/ui/index.html b/webui/ui/index.html new file mode 100644 index 00000000..3f8bf226 --- /dev/null +++ b/webui/ui/index.html @@ -0,0 +1,976 @@ + + + + + + Gickup + + + + + + + +
+ + +
+
+
Total Backups
—
+
Successful
—
+
Failed
—
+
Unique Repos
—
+
Sources
—
+
+ +
+
+

Run Backup

+
+ +
+
+
+ + +
+ +
+ +
+
+

Backup History

+
+ + + +
+
+ +
+ + + + +
+ +
+
+
📦
+
No backup data yet
+
Backup entries will appear here once a run completes.
+
+
+
+
+ + +
+
+
+

Configuration

+
+ +
+
+
+
+ +
+ + +
+
+
+
+ + +
+
+

API Reference

+
+ +
+
GET/api/status
+
Return all recorded backup entries as a JSON array.
+
+
+
DELETE/api/status
+
Clear all recorded backup entries.
+
+ +
+
GET/api/configs
+
Return the list of loaded config blocks (name, sources, destinations, cron spec, next run).
+
+ +
+
GET/api/running
+
Returns {"running": true|false} — whether a backup run is currently in progress.
+
+ +
+
POST/api/run
+
Trigger an immediate backup run.
+ Body: {"index": -1} — -1 runs all configs, ≥0 runs the config at that index.
+ Returns 202 Accepted or 409 Conflict if already running. +
+
+ +
+
GET/api/config
+
Read a config file.
+ ?file=0 — zero-based file index (default 0).
+ ?list=1 — list all registered config files as JSON instead. +
+
+
+
POST/api/config
+
Overwrite a config file with the plain-text request body.
+ ?file=0 — zero-based file index (default 0). +
+
+ +
+
+
+ +
+ + + + diff --git a/webui/ui/logo.png b/webui/ui/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..1b0d3e820e1b1d03c6dd14bb10a8531c0cc89df8 GIT binary patch literal 35084 zcmeFYbx@qmwl_*}3j~5Y1Pku&5>|kzX3#0;jI0C7F?pEe-aPISmsd|YAwf3RQypfmzw5W!m z(zq+1QY1sOu=nN(9qu<~#Xfq-PaHx@4D!Fqtn~SPdARX=xc7OAq5PJfS>jVY_i3$g zRA^|eN8U($-)FeEHSQQ{kOe(+SoUl6Q*&2xPb~L~y_$PkxWB!}YRx1lzbGV+Ilnc2 zz6$y%eoxtUumhrs;AU$6360&dkke=t>(3%DgQnC9-a(aSpDt~)j$42$#}Q$yy9V;N zwWsXQkc_<)XR^0FbV_UL8z6XGzq9@|*)2}7t!AKai|=w$o$G^>r+CJ<>pIkf&Axl6 zgV@a#_wIX^oTt|QHrLaI((M^OU%yKw!cxDbB%{dZ-J`B3@dv7XO1FKx0acXgyTj-~ z2XRZ@A%{dk--q8e>#a8CIHi1_#>B?FZ4%lJLtpYoeZIDdSsu+gYiw(aUw$!<{i*^y zv30$Xf)UoL3Yj|_c?4l3a%38yAFVbku zQu2)xUS7`-5vT<6v8H4IThFllvYz+4+RpA5Np+!uF~>D)&4UGiH*n$Au1e z1UC!Eg^#-o4~W&wX|gFO82VOzhbA^p0jwVPEVhLrek10-jXtgt5dyW&TpxENtUvP9 zrF*^=wb&D|99tWX`c=6r@qx8hA(3ffaB_&z$+mU!yvc5SLb3Vfz&PqR`iv!xAfD0> zNa8J69MxLYCDxOW_3`3Tg;Etgn-(NJ2wRKm;g4&vy91m4B565NpJ>j$R3N%pN!5s0 zUpnZ=84FhR?2O5Mk8qRioXfs_({aVNA<9j<=SB4-RN$P2nI5@Zx5~b&fE5}i&1z`S z;&Ht?IfG=QW8dPraOk$^F>?*^X}oJ%ZX2t4*!|=ojK6SsY_R^j@Y+(#V;br;z`2GsTpz3qsx$A)qdR~D-vY6O)MJCsWqLn+HtY_6j%THoIc7rR*2hnUM zF`3n!&{|#c(ZsW7M`)WopxcHYuZkq+rRPg5FhGWPDqU^|)AuD!BO%=%ix11d`@~dc z<@V^dQ7KB!y9S4Io}-3USCwPok#HRi&j`GO7QQVPZ(NqE-C9aP*3dF&x85#OxV5%0 zTo7usIZi5O<7Ym*)2~j0R-iI+#H5egFE8YgDAR}8w_++B*VyZJYx3(5F@0qB5iGc| zu0a{03=uR4s~{WFwCSAC$(R!g9kOvF_Qnd82we!2y9!tVQ+5p@@v;nI)74{4@03;* zBk+s!ccDpp8O!oR zpW#tZwUvPEiJX_eTAkQozz^k37iN_@($vq*Yzc8+tM9S$(g#?(xN82aH^y#7yiOd6 zO)wzT7Ku%QsB}|s_F|15I5bT8yd|wus_GrA93P*Y$Yi@t5RKdFN+~-4cCP%k)Zbh0 zmwroX2seB*Fcb-w4fyyj?>RkAj`DVAtE+g)x`4&$UO(Mcef{;$4Cg_uET`vyf5n7~ zcNj-!O%D-67($t~X)Qa|ScP2|JbCVHP(atm!{5`Fa-|zRpPF`BcBP1ZXi|&fzumVO ze6LbTaYL=>bitM;rz>c4Rkq@F`%T(VT1)*Ma!mZCt4S^Cy>iAJmt6F0(;EeD#}A|C z{$H4;g_F^G9rFZTzvR8Bf2S;jS~ZP4>7$p@BJ^~vL80bOTw@Ts>H9^x+hbTb!-umg zsWH&DgUgmxVq%N+9xg<0KHp1g?PAbT%UkB1-0>^HwYvROKDw6(dSnC4BR<|)3(ZTd zMcP<@L>CL={oR$S0R&pm@Hd3Ir0N?lh7frrR7Za z<8(3D@z8zE7N3R)1MgLleO?8U9J&jFXu?)%2Z8@+0xP5OSSgC6kjeWq4L?<{cg{GeLVM4W%X#47Ol$&<{Z@!3nc!yF3 zSNqTCwrY9Z&eRraZ-g+*Wm|Z@XG*UJ2aO@j9Wp8%b)@yMXG{4LXv9(L)+0&YXIxZ*z*F-rG5S= zl#g9R)tZf>Eb?=^?A6RN-mC+3w?odgJ|@1yErIVxOU4N1yv8qQ|dfH>ej$8#X z6stG>R=6q+>4egHkx-2r?_pGBDuFMqB+Z+&85BxzC~|71qOAd~SBse((kn~kR&bmn zn#tclKr;a)-BfLUEGv4RI}#JYv5t`9PDHF=2zo30>o16B%QDa^&$KX^g}|d#5=WlL zPW(6ikv#3hgKw5I!_I8jJ}n5eon^{}Z&lGY1FVuw z2WM)f_sRFvitvdpRN&hMRWZCM&6z8FgQ}=Gh){lXnr)LlWm*0do+E5)e(VnKP^s~- z0SgeKl}im9^wahsMA&KMX}po|!nBp!JQ@D=`uvL%r4&*TVkCEp@m59%!LT1I8j2u} z)mGnq0T@aRiopP=y{>o8V;kGYT$m83Qxj?yEc?c`L2D3*pRH0up83ja_oO;=yTx`R zM&ny7F}qAB$yp6&jRM1BQv#J!5n@J>|1nl=N2XmmS%NbY`5|7~>%Cscd3Q!Sv~x0l zJH0P`*3(*a_XU?0HaaH!z1KG`Z%K4QL3vK>Sk%ZFdn*$W&z~jjF$XU<3Wg7)ZnDoM zXn6x@K6ivldSiBA(|A4c<;hl`*O$T5C-)ZSey#^ZvJX?Ju;Frdx%Ry8h|A%NuT+o7 zMt=YZL4n@Z)2to}KRZ?RRK+-1dHwMDgpfQD(sLWGjIIqNvfe|j%spk#N6D9TR zYBA-v;t2h%a6qVpw44sVHg|b)x`{oHiyY;MzKd$Rih6ZK#QG}P zSmuoD31zG^pB11^wYZM}r1j-ELF)_CeVwwsib+Ev6vKo;85Du^d4nLkL;$4G(fXRZ ze}Iz`Yq!+@5~EK$Du=9GGWZ<&{j(NlCsG*sx*$TQb-FYO1Eh>Irk<8SU^T6t$ee%; zLxLpb^}BZgZf1D6y%D*pRT^btIN)TR+3A3Vesd_<;$- zy|^76*=h1O%Hp+WJa=E=1P-zyoRPZ_E84#&@~hE~D_kR%bFDbZ`VirF-dnvNM!m(5 za20U1XTczs7@=8oid4j%gdj;*kp#a(8bISa6;VSFrf?$vVd9@nk=tdhdQlpy=8lPJ z^Ul%ugRnYA*=a4RhOU7Kn}=U7W=;nJ_s{^&9pSW2zVSP^#k#e&b!y22mk^&o@WAaV z);hBCjog+CSu|)4tCWR+`c>O@Pzu!4z&p`H8Z zE^!@+E3xi*%kA_{paIS?OWHYIpLG1y2m~r~SMtP58)t||gUWXUybjf?k z43hG*kVumr-jY}aL@W=V^Jg9-u&CeHNMg`%`1@h9=kz+=bfD>su$RaqTe8gEcPE$) zhZ8nGP`@!2TFiS)-2WVcDr=b2{!=iR_p~3Mx#i7%4s8r#72kALv8}6PhSO|4)zljN zOgzFHTD5oyRIJZbohMb!J)o(%w^#uJs|_>)R4tO#Zj$9_H;d^z*cIzmh!Mej9W`H~ zPZP(|LMkC4H5{GZL~hme+>>*wKe?QK%xeC{;vAU?CGz0OO+!g9-VCMQpc5Qg18Hd%)7_WuC{7MVLD)Ul(SDuV9a>^_s z6bUD5`cX1MV2k-ki4KzoV{Z=1($tVGF_lf_4}*i8d5{C@0RvrF1Yz+Tum__$!u*8-aA+U&Z9eq=*np zja9k&!^JD`&koZ5buI>(BvNKJB!mQBQC*ACg-+2aS#vb~S4u%-i2oVQj`;2Q&x3>!mjhM0N4*D-FrIt%(T_L1In&s;>%?fc6? zso@mTQF!18BcA56>Zp@~i={{S%CK&zga~B9iilY)3K#NP2lpzIBf$UMcoBn{liAWC z29LO+ygMYEE3+wtQOE+ZlU&omX*WCRq{#Xvl2pbA749AGgZ@WTXdC*#xdlE-l760N zLv{VMvQBuIB_joscC*0)C8~6wO(2{8Mcw8{GmMdV&6kSfpo3{qE=Q5@@}oJk2wegMjPc;jw4g6b*sr zs54Uurf+D_6(`C{+A*{(*MozC`XI^Hy?)MliFFfhXPJR)|?typcNm}QCCyVIaXeAUgmsx-P zfCKa9;?2>~m!{!)?Tc%U0-$h-&v{}4GY2$m+BrdH4+!hV@Et1pAq|MdqWy`IjqU=} zW3S)yud#2qMC=nKk6gYY-|d^3l498x;Vkl})XxD~Xha?b(Qlz1VvLsK`M-{X8*)YY z+A#%Tfb^JdIL$B$tuXNP8D!~2e6p23FAX;Zv)T_TFEXK6O2#$wl4@dHo2F47xDdg| zEC@TVS^MU%lWR~nhfU@5ECl!hts>Mu5D_LbX<)zJH32JxRk|q&$7;7_WJ9{5h7-NW`pqK8y#G%3OPZf>!(2pjFW@IKFL1g?+A40< zB}_iVN)$TbTecb!mR+Y9b;cyD74f}8vZn}oAv?<1`(0P8Z*lo&c@b+jfy0Txs7`=@ z6XGd6u9azHgcNh)D@!0-s;=dh`O_@dY`OH%jRKdBzS#FB863@r;K0TDd`}39&5Mh>n8gHK!+EgFa^q3 zfgEiC9%|gMieI{^v9X2%3u8Y3U~Jq!!u%H9*-axnJX*upufH8DxEa#~6Fp*}2PyW& z3o!{gOb!x!@21{%)Li!cz+oXe-Wr7(!}b}If|IGJa+x%asXRDH!ml+msz;P?&3KGR zMIs_1O@G_Xv#;>4nh^R4-ogu^_h%xQ6iUyMN*vIl-lOWO$baVk?1Rw%L4H&oF}33c zp?_LI3RhLmMEVfvg?-ZST3~v&VhEfmJcJG*;roXKf6MKMz`@+%i)dUVIb+6rCigA6 z{ar>PVxm-T4*yQF+&d7J01I)?C{=grF*nET(KGC2ADleniyRswk)bb)V?siJi5a+v ziOJwfKubUu7?K>2{FXH)V0F?N8F@(_8F>{$LUg>xs`s6-$2$rJWaMeD=V!kUFhOMv zx1YFmK0mh+K0ZIYajKn@BZne*z~NH^j}G@{oM2VgI4f9nRaZq>$kf4(4PfSA0%UWy zbA(l2;owBX-C@GF0fMPafEHHvqBO@%Z8TI?W}-AYyeb?jj?zF&D|t_6pr+@0EmKb$ zQ$aHtaWQlecOe*n9S{tla<{X!2MM{0()@)h1pEA_nVp8}FA}hgD2=X)8kMwzGmwgh zjfahcRo30gm5W9Uol3;n%v?xA=G{LaVBbV(EWuz$A$E2*H#asnZZ-#J3wBOHK|yv7 zE_N<1Ru}~<$ip5CaA&m#z4-(2H--!lWa@0?2)1&tr}~2lFmZ4Ji_*}*`l* z17!w-;s@~Z@d5;RS$PFbO;~x%`8ZhxdH76Ofjk_1+d>;X18RZ+f27ID z$Is2pEx^UW!^asQ3`M=e4yW?+H^{IOFQz+WT&YFD## z06W0E#@WhL!5$2Bc65e$(BD*l?)^c#L6{Yzj2dY2Q{mZAQMEsl zRslX%0dqkBD=#09pc&8%U}7%tw}t!@9pqpRb^|yAB`sj$g>eJ3+P}D=V*E>gZ~yyT z+$@2Al*7Tr&C0>a%E_U{At=PjA;iJM%)u$d!9m0R4~E(Qgp&W_ScLumB@>aq2>vbh zz{dUE28(I15Xt_}82JyG{So8;#n(S{@qh6OFzEkWu>rTwH+5vT6lePo%+Cx=0f{Q8!W@!P#;K-}mQwQmuVh=ws^@bLt0qU&U7y{d>Az)HaDS4gpL>hlk6XK% zySi?P9G8oWhrgnh3P#1Dk@^4chZANQk-S&XMG28}j?>{+r^9*O+!nG9!{z)k7X%M+ zG#;gp;^kdiD}e_3G-`7aJS89&+0k|1S&tvGnEcyqSqCkd3)Ex7ujQ5{D&kyju6k9F zebn}AAMfQpCj+eOm-9Yd>sE?_%$`-gkJ+*g9nYv=p?kK)&rW(E)1ZR=o6NBN+j>aU zek1CO)?>;Dc?ulS&ZeXdYdz@^w|Tu(lHgqb<8=Y7*mdIttjz6I4~h>xu}h`lJ2*=& z@@AOcA7;L<{XS%kVL-R4b@HLiVML`?)25?sJih<9wo8;h;C!+5cqbXN6zG-_GPrws zW&^sQf4-wu>}_e1&z$DDtZ4*Gk&~o71Ykc?1>X#Jj@FjCU6M8A@O`t);BR_VYC#BE zFMwD~76)^@jMw6deFZ=BZJN{!chp66aFZ)CmkO_=<4w@IX?#z9g0oyMzi{Oaj<5FE z2&sQBQpRB^0l2zcCHamuuR+1bILAm=Ctk3LK5g8(a#z>KE=In4-B-jMof?sf z158}(k%{e}%LJzjjM6BM&S9zu1X{Ex-nQ8)e9fTN!#y`c>EQKqn9;-fv-i z*Bmfab(HPE7_^YYj6J?5YK+w`HWtQ`?#jcAQORaP=alaH<$?|@#pS=ek`@gM&Epq= z&2Y&BmwCq=_qTqaeQkEy>z#s7`zOxETXisq^%ZFvi}!DcR^rUeR5dy^$IErrEDaoZ zYDD+|wT+acd*fu}lH9dhvCMt-N+x9j)_3$#0-1#+kwPDaRmI4C7{AEIos8mmBRL8? z=9DmvIB7D(hja6N{H|Y7ch}nH2(ZvAXvm3G>?zP?;?7JXv$C@AI%HP0_~mTwC5Yki+TX(x=CgvAlUQ=9$J56{lsy5C8S%prBplq=S! z&rp6}bq24k-DT?y(duw&(QY-j@2$vXd1xY2g!$y)iComy=(K6U?d#)>dt(yhm`fLj zGj7TSCt@;^oNY(S+QaPNu#mW9S0Y9!m|j!qvE#N1%oRVRq}q~btE*|_cX5opBwuh+ z=*HA=s={!Ob~l&2aFUgYJ2}@ouPFG_ z&eVtXG&urw?YP8+Fd&~_C8*vicb}vH;JDqQTgc8Ax{bP6BaV?&viqqdTb@Z4<1K+t>L z0SA3oYN~h#Ik`R$r)|U9=Grub7N3Jzzi^MLg)tX$ZS8TMmqV4awIIr&rtPmPg$OnW z1GMQhQ2W(;_sqBwbWDT3bCFwjBjYAEm5~uJ*wuM}F6rc;K;^q88dGhI8e-dJMVy9? zPG4S}Xz$hG!tRl!`t=2t?{pADp6Zc-XA3MUmcuoEv>=)9`;p*V`+~qqHwh;^6OD)4f6@d)8eG1q^2;n$}YN2vj(6tGC(3OsuRw z$_(6anUmQ2qI-5e{q!#=c*iGpOIuR1BES|WdWyH)d`{|{uYcjtx@*oJubeF~UG^Ac z>K9;oJB~usaD?N_cR-fONLIrV6>v&+Eq+S|ETw_MsYtHG4-xinVlcw38_*}FbYFutp{;58 zqm(W~S&i}I1$NA5X1wHKLTakkk`g>6I+A^B1$tb-dK7NLL|{j?W2-*n*PBp?QsufS z5%T_YhEy$m6pa?ROJWlI0iyf9f@5GjFDN=TCdzpZ_tZW5l;#S|~O8lGg8j(Q2(j5UTeEF4`K@(rkJhSRCK%K&cppEU88h`>Lkp}i$;cDuyd>#~C!zR`O{^E|J|ngg{0vTImTP~)O+^n$i;@uFlQ9?Lbba&ZnR{iN^?S>HQ#ZF8aCG5kz?92tu80#&^iH$sGhF2avldBDCqi+s+ zc_Pc{+)L1Z!(4PO(j~xO!1A02VmASHEK}bfx4()>5!lWG5g6U>x0&Bo0%2E=#lkYt zqYJj+bQ`rzCE1henF;wF#bHWcYk($C5nT?SYX4cjK=-Hd3z+=}Tftxy`OMgMU-f+`zx=?NZt75X(Pv`?m{6+{)u#80Kx zOp&h=1_=_iJAU!Do_#BaJMnwEiJNWSCEgm%7@Ae1EH(>loc|;OHMKOuG?>w?wB)qN z^x-Q}8{>H?3%h$>LkoYVVq7aGaqArxnSg}RGnJHjc#jFARJILM34GoyF7GZPgrQJZ zMa9NR)Zt4Z$;ai|+6QGnkO((v-yqU52Kd}&XNx0tiqPQI7un~UCy{sb824m-9p|Dv z2Jw$anZrMabUy_HYh$lIC{J#*-XE<$-5+f(Rzc0CQeV$iJ;T9gC2rAw(n~#oZ4FUP z3M5T6Ux(7a_K39$yS%a{jHba0&dnx8PBQey+10`gNC)Z!5??8E1>}^}Lk@b+gSOJa zivr7+53Xw|H&BT)S7U}#WgEw#F#EQ>zy&zbMogOA=TCW6`M$XgeB4SiJcwT<3am_0 z=}Srmops07Qk7{rRDv2Zsvd2XZ+^V|7}#jZVytwGZMC2^^<`Zn{y#C1~uw zD$*(RuAAYW8jl)uU5bz`0CLrQv2KbP(y*-DI?ALZ19z?ICXy^b+25Je$Dx8IB7zyqy!OC;XEu~dVK#~^X-w( zj`|07z2VAr0(F>&zCw?`711;FPT4l{rgK~dVjOhe2&mdF_9y<(6FjV*Jn=aZA$$e% z>2;xGb4;kqfSC;=!C8V59oHPoP$J7#q7qbYh6k53{x^!}&3xM<9AdZf>lqg-h(ZPz z*T+S5YE9L*I^Gy{EvFmWLxd?4!{BA-DKB4X7v=*Y_nTd%QN!nAz4C^GAwQ4pwnq=J zwv5d4rIZNBY33kN*v7h&i$c!-bAC-puU?H{Z^3g9 ziggunD#`w-C^jYK45)%U&EdBTVGS?u!9p80wVC|q5%Z$bA26TBlarizi5M7KK!6-a z)=5lB|6u^sIU!3XG6Cgpy_bfxuGr#62)O149^V(GDhj1!+s>sPy4T&D)L8kgk0=T2 z@i)&;K9jZw&e@H0)(6_mC&INaa9L=1RU{2=TkEJ}-d+H-SweDnK9nb(g(2dLeQ zq3<_d!O=yJbL&Q7i__WHqz#*mr9vcJ3S!UvZV$3h`6C7JIVQLYzfTCyfDvbWc}m`* zlHawO*Qf}22=Br8tSr_2wUly9w&YK%kQ0l^wvsv>D7vh=6hJ;DIDUYC!S$yd>Ezg- zC2{stsJ!h-C^Gl+$TUnJL=+SSZGQez)T%jZ?Zm=^l@JfinAp?dq4IGxy&)wptijnu zmxwfTM_eR71Ye~NeDU@Q^cUmzqxvlunn{@z_M_+Qtvt_>=TPg?k?$$or7vOW!t^h$ z@|JH+ZC(!0Edx=M=bf&Z=b@K`Wo^-Kw9xs~PvSpk zCjv_uO-eSj>EgC;Z*Bl(5GE#$8Xon%o?GtbfIaG0Y)%rmhHwKiWU}f-h#2Is?Py;| zkZ8K^oz=w!*#@bbZ&1!EfWm!8GIRclMEIK0*td_xZ~^4sP_-a9#vi`l5FM)mOF5(@ zTGeQE7_l}YH~rl7kI~4KAXYAAUrWMkbZiA|oY0%><|fuTOr3aC-o?#WU&-t?QWF57e{Sg?&o#_AHFkF;yfB)_H}ATKTU_u z+kg#TJ5#2L(~Tc^!HO8D1YBBa=HD-oRpLi@QVUdGCl#Sfy^oOkz`IfO>0zaIZ{Dm_ zqvNu~6Rcrlf1N3chwdQ)?W(oY(grv??NY z$ts~Glrmlz<=m^9VyoLtX4BMOn3cD%;pR8;cv~F7>4_5iRe{b)^fr~jIbIp>M_z*S z^C`U_=vo@uR_SL6PS?n3FGw;(^V4D@c!*II;k<1SZUde3Q!KiB$(*?nsCyjh1S@l0 zZBB260!vClOE`R|^09D1Q{8E(Sn!m^_w-VwENS{BLzE&=j0hGqiu!|&_?uUIt-_F@ z)quQeK7NWSfwdpT+Dv6Cmff}XQVEfa@QvG0>MZt^daMDnuaWCFkbSyvi)v!C7>fk# zWyeEtL)X!}OGD-DRh2YLd&`xTRm0%p^W*EMlbkj`_sK&C@bw!R8NV4R+R7O>m+ECh zZHphcKj%B>>++S+>a}ckCb>^1XcjC>m@Ec@NSe4}swJ0Wn#t+PM%M2yUXu0jfJK(& zUixmU25x)&`gTT^^oy8uXM23c_1qcK-Ap1+yOy8`q<0o%!q@A@JUkwHoyX1MET^Y;Ac9f$f=N>ByYNk&wst~6&KFLS zJ)3bnI$HOgPMslBj=V>0s^6E^Qudy8=_IbuTTOdE-VfLm$4Ex7$fK=p>0oDW>7H;X zd^I+qCdU`w%M(Na2^pRM z^Incp^pM{*KW!via*J+SCDa3Zsg^C8)i$b(o)(02qv&Yvn!c=9txT$<%=X1?4O&Uj zpqn%*E{>@?z1=wH`85@am^5iNMN*G0*{R~WAeLGZdcNEU89GKt(wodDJwkbE+`g?u z)L~uFR-kx*n9<;DA@A*57yTIE;^OP=SQ0zysnlQpZo|VrJ-##%y)KY21OyvMYff23 zNTbYvh)J_nB$mg;9WmD3-g{BvX5Eab_*Feca}$8w-13UhTiM5`$O=48dy>egiK5w3 zjaYKe=7m+y&i%utS!|okWZlA2(pbiLOTWYxkj#@*R{sJPLFpA3Oi)LtQo@ILdmv%2 zvP;lQazj6~Fs+v%9hOqDdhiY%dQhaY6)rriymAy}XPMO2+-cob5k`IZyphSkeFxe9 zs@!c=RGO1tkr~>^lA;v|W>V;>eVa6-n;4QqT3I^E1A7unRrzQ6@eU;N5LN~VjjJOymIIFS= z>1gBkIY~v8gwbAi5rbyFWjI8Z&^Y1*C>ho4O>MrMn)lS+k-{f|>?5xRiEu>rXv_+t zhyjPt<;;ARlM-mzP%{7O#E`{3iVV zk>fO3p0=kWQ#yV;@tVR_F|p7qn*Zv`4L z5dK+arf(!LEfsv0Tu4ro@x8T`Vsb19cTS7EDfJbG*V1FL>hN{G2#8m38|i!6d^=O0 z2C$h`&$>h$WPlCLV(OWzjma$nm_VJ#%PK<(tP&LSv$?4B>|aVV5X22F?|SGa$`USFyF4h~+PK?9fE% zUmM{!@Pa`Ijkn)w?}m+x{B9d87M&k~yH8I~iLr^HX>rj6$m1j20ZlV(wl;?Iyn5(N z^l57Gm+qTraMlwFS7anq_9OsrJy|v*GZr0VcW^YUNGPiNFp#SujghH>2XnP;*-sC!i23u-8$qFqQO>OVxF06SIJK>udr!ijG zCMKe9Jz!=fL>$50cj&`3SzD?Z4MFcwffa5PL+7h*j%Z`F4J{2ZeopFE-|aS$KvUVv z^M04-{MXGdZeFf*6;igjckD8h=XiB2a*+Bwob^PjZj5{xk0_5*+0d;r!>UQ4MGg6t z?vn0w3AFRreKx`UHPZ6d`lEx%nemBV{sk&6+W8tOZO_v_)iZ?x zz61JSonG@As-6!$-qf#qMN{3bn0;zk6RpTG|KU&-rFei%0g{7^z&=qqj~Wc_`{A5C zK9kX@`6%odB@H)hUA(;`A3GrRmbU4zeiowC<0=5FfFgl}rI-y50p*Ey6ze=Cdw3Y%3{o=3A*zC@SeyrX2+xPl^%{;+uhkh7Z-$Q_*)@vsY^KSe zYqC+gvr|3kf}rpe`Z?P~Bn*O|e$OvFJw4U%kQQ8}#P!h{FM?lnuXA2e3U!xVxo$zz z!ib}CtVvQKLw6o(PWAhcrstDn?W zK}wiNvpg~(x1i*O@!0)_=0;O??1vo%bN>8~wQ5Yn7_t^qp=kow)$7I3enjnWD-d>f z_jNwshn4HMmn~#sZgV9ZB76gAQNpvJ9mb3wF9D$~x)8MrXIc~b$djO3kU3`AO=}$G zul`cbMzU*6MVnYyK>)ugu%wu};{|3{w#kBI@yn-c$|q_89N%FOmftp1iloxyqK)qoZ?}`i)55itFT)uiBv-rn|Q5`j|9CA^l(cV zzt;bFRCS16s#AWamRoyj>14}@8yFq_6pPcm-#Hogd@<2uz(1PCtyfw)(z^Rw$f9ms z*J#L3tO9MQRQCX zrZ30$o6#8_qMHD$C<$?~4HLhA*iN2WdT(Grz6(azp6WN_f}@RB7|j2QS1|U-)F!Rk z?Lvn4JG;0TE$|@#Vg)P4!cwJ1h^aP{^ZTXCS;x^ik(05m>RO&E?;b_i;cmEs%SRk{ za?tj$f_r8iYc2qm#^D+XTo1>yJ2==qMYvU#p(9A=ZD7z*_<3?WO_|ENZFA-3}kC8yR7vx{~c|>pib-fhipa5QQA|fLLSeivkw>&o2ni48))qD+4akve>ZPmHGay#1O7k7UneltEP z^3+~e9DRiCCnl0;`uqj3Rm(r27N6>*TU62+d4yHl>ic;#F_hQ@yCwxs40ZxSWFMOY z2rp){uS`_zq5pFhPiIn~kfS9v`LeOGac5_z9b{mC(|+G|*K~ZpcnN(TUX%R|5M4Y6 zsw{yFi21#yrazI-Fsa(vQuFLezKMqBNQWuVYi^w1{>)*Zh|^@mI}wdbvu_T-ABugG zG2mEKmoz$x<>}?MYgmn1s-H-FlYMZtxV7CE!2XUW`m59N$`^C8UngP}_J`&_cHL>8G~vKb>!l0y~VUwuBf1%s=3@xvwBaB;j* zpNK~A>yN(13wlg>zeC3Qxqq2-+YuygLZd^KHlksq6PC-!!^h|UN}!fC|9dceIMo70 zzvf%?P^(aBP5^(V^1;xz!AQjl-S0K|DH%uX(hL=VnPjWTw5W|{>Cd!80J!V*SkByuZ zI?m4_lW)Q;z|*;|>R>MEC3?xb%kT35c0%s^#??2~1v?F~)l2NY3+siGo$g0qjHJSW z9e%9+GRNi4Jiqj5Ne-3lag}smJXzwlaJB3IGSJFpm3xOS|q;o z`TV$xASdUsJM|dujzM{Q;!x>6p39N5E#R8$p@WCCd@fBltxFTJF9fS@nw$oU5W%~( z=riT+Bl|o)YL#(8?Up**mV5|9?SFdDaY0(Pd|>gND}@DC(Kn%nnz8lC*nf>t74K5v zUy^5v@379&UpX2;n21EC)VjIex{CZQX!=VD`0^!Rq)Lwoe01@ZxDH= zf#WVQFv8(q3Wd1c${Fs%X8TtyBGsorp-+3oL$LJ6bakKU8?&vii+Cx4o8weZA+BG% zV+S^PPLAO$I!0B4xV=9xaOukjbQ z0$*-}dUp#~Jt+pva2IL^w&|18hjHQa69aZ@+r)y_5Kb)Pb1zWHotjfg*)^cZWhmA~d+JDmD`9 zBU<)xaa5tSaW_*us&O7O{+5Ql*)u$f?HP_CtC|fwlwnLWQbCS6DKP~e)+Ot!m0lCB zRq0{UWWvXhnjW-k>byEE4<7s(MZosGxEP172zBK?K~eS~@&a`tg(URdyB)#BAq9rA zj`JLSCTD1y-@;=H)CWM9zgUWcpJ%qXK3-N6krL{qCcnWr9XqyvJKld6eQzwNJGmQ4W*4p_O{U+ zATl72p?ytnLr`m?tq)eAOJZO35bH9H=lPYtl&~TF=C#N}H-^Z~0oFW;_03*V+wf^0 zrK$>_@W_N}#D@n;6${wS9GTK#9=<~aPY?TUmEOK@EHazZxu;o8W-zcTL47u`b%j5; zG`V!UW&0M;`A%DVE;!FRXNWS5?;gU4e!evVbaxMK0B!ky4$CO9)?&d>nF$0Wa+Q?i z?h27tS>%oPPNI%{XHICxUK zu0@z?yUMmOi-S44ItIS4^GxT%6{!vxtX$5~hTX*v+Ygu!k?^Lt3xpIf25QlSLQtLw z;U|Ad)6FnN0>gw`TT;TZKELE?@c#JT--B(&u+|rOKon|mt|WY0cr2zM^u_paB7`XP z{dH%ESx}=JQ9Mc@e(N?$MBO_QQqqs02Wxxx=ga2mLtv}Rb=&EpQq$~DU*1NsQSqnW zM$YRf;-UMVw%%t7usx(jrc)!C_o)caZ&bJDw_)-Qly)^I&&ppHjTK2b z{lp%%#Ty^J|D4~K#1iHWDEQFR-Y?Mh*b$}BTex-^d3e$imbsc^;WokBgk&%|#`{fK zNJWJ&YP&%ziPAkFApslER7#N8FlTbSw01+(qg!}8|N<)`1v6}~RQ=X&wpn@U-E z*T$4}`aW;{p4pc-$5y)*Z=UwmT}1J|mW6VsvNau-e01;P+0;r^b4o`qDG0Os8CjBR zB_!55yoM4Mw%=&RO}ymQvng_O@&@+w)cE2e*g~cQ#tb0})0J=?Lbjh}`Ym}3_qxT* zo3=J=@2o*fEN{x8pz#WkNJ!omA9*Pg0XXn?T~=6Bvu9aba~yD>JdZh&2-fpK+ZXrh11id3w0twBRr@B6qVuc$B@TaVDN=VQUiOv&>-*{WjN4G*Xk{*!^l9pq+d?yGoHJ7n8wx2_1cu>DZ!{ zCgMMfee$2OB?C1z+g>_KgRTyoO0+3*?)%u}G?MAW2j&aB|f>w{i0>fdo+8b^=IY`nlu)L)*=#x~q`OiA(mc<-TI zX>3aJZPJ_JOmhPgjr@u0z=gHX#3BMibd1CzdNI7hWBxRGsV{;We^Cql90W-YV?iv% zpMNlFg%?T+OVeUCX^44%Pt|jDsbg<*-o2yME5w|foMSGcNM6lIuvi05{Og>S4Aqnp zB%6-|gEDkqdd%S+JJ}Ds0|eTW)}9v^7Pwr8Ei*1QJ1iB328^MK(iBC|R@TiQ-z%?>g#i|eQzrYIHE`R+^^QNGb-rZrRDpdQ z^^X2_=gF5%ckR@Lk&7PCQ=SbB*aph6@;;&MFg>J+HBnj7b^;8tkUu`w$Fn(D1YrzD z`{}WN9Mfx8>yHU}zdt7i$=@OuLtRWX1Y6jo z!AHxlH3U{ZB$WWylD3QmP@+U|iX$M=F$EoUMUh|>6;2}ngNU2`qHeN)G9rG|leq4m zDXz^-XTO)06rx3gAetdhJ)>@%%x~JSh)$nR&hImEEY4F7{@0PplG@r9Gr^{G>5P7_ zGWdCPT@@`slpFLyj1j*Ln4`@roudqT{`7B5Q|do7`CKq6 z4k9e#cJkIL03BRUzLnLR4o0~wu6nNhOig4m|B#4A+vP{yHK)Wv@;#G>1vNtP(Tb!- zJ_IHehlh_}9Qn*G9sR@hx?B^RR9^<#;|2x>_Gj-`EPU5o>dMYd(Y+T<=eNo8B<4e{ zvb#Yq?>O6r(siXZ>W&FwIF|L>mmT$}{Lb>>3jk8Wpi14zvJ~!U+w2eII zTXc0h7;qGb)`CNbU*R*?7o{mk(1f5?8^HdXGzKk~#VqUDjYV$Ha=O?vHf>G$Qi`6# z>*ZZS;KK<+f3!Gx5=L{aT}R?6uTMUqjyGg&f+^nU0ii*LM5njQ=K-!94cXG;S9nyk zYeLd@#2m0h!M?qe;DYsNm<<%i;>*Te!CMk$G$G zX1Z*7+e7oNn=%%+`Ri-!NDFr+afE~3Pf7H!jStn28^R>F$(NSGfYd|ZJE9}gMirXF zOp94CoMaTXsgzbyLRm6&M<@<@y=F8g+w-Fm9$C=8-6T#lZrkW{eUJ{W8D@%yEv32-V}Rb-nMfJQo~gHg24p@}7Gd(AD8=1y1#`y^G38Hj!*!Q; z*-n03a_0}ANkii7!*c<&+mERY=sv^XAR!(@KPVL2ypGg5j{ja3D5y}~cMy1*Exv=MfMp2`htNon%l&#MD>VnMcc4#>F7S}6vVfI@ou;Q*4= zR^L6Dq??z-JO|%vYU$CD<<@u2#XKLV*N5}bdqJ>*Yueg#MbYVfxt$B9_BP|D*wR_t zY;5AQEpq!M;GZ_sS=h9A$C7t(ekl%B)cj=?)cWmQ(pMxECu=>cf%*KwSvYpg?-k3R z7LuCcXlHY7I4ZU@19t}0t!#k) z`7)coguyg>&-E8it{=a#7tq)j)ktgAD!?uQOKu@N>KZ*&axAlJm`JlcV5#2Wbw-8MRZ8z{`2|CUv6?`Hq%_UZ42mf3~PG3u8cY*c-S?l}@XGI$>#egqRy3P6Qs}X z1++=~>sO`5&o-;znfc}EsbcBrt+qcUKW;5!kCjq@$tH1CjoQu!JutfgmAO8HPlH^g z)8S$J^o~flAz`n81h8cn)-7_oLZGcP9NW3@by#nWlU&;yYC5$lpB$6;E9KL#ci1%7K%?++`SQ=eYjsISg&x&3#Eo#OI2CtAJ8oTKCbL0&|qMP6U5$ znl82nBS8Dpy8Z>DbqG)3KAun-Yc1`|9 zF0+-+#8uVc`kBWbZi>&fJj0CX3CdvFT_mh~bs*~|CI>&EL0wuVnlx6R+u-%vRl_%> zfxiJ+b)T!dZ|7|V@O|^x2a`^M2iM|Vo;1O`2YCV8iKOm}wzSh5YM+3|Dh+zY_#}>B zEZYolu@nQpROJw)CPOK0?Cntq4J&h0s~BS6UGf^gl6% zH#*}Xlq;F@ANCx(u~gU(#fWn~6xkd`A^pG_EXK?P^aHK)W48Cdr71LaxBiK9atc?Uwm~=>YkG|IRwjyozG8b7$-W7e z4Bys-v)yZJv{SdLESIRtLQzP!9bf}^tgv{2;(E=3cOk-?HMr5|1&l0r=7VY0$26XejT@5qa)b4uaz# zWcmPO&Ra3fv>(TxQzbhAAD9`pWx<~iNrJ$@u{;t`bi)g-aFDQeZtuawW;M_JjPZtR zjB@a`e{<`^!I0*9q0X?rshL4f?o1dVvx-KTma0_VKSCog;5LaVq6^h$9F>~TQPh*s zOyTkzO-Erb{hD~HLXSlUkE65D9O6SkwWLvN5|-_NGUdD?Kc0I+cyxBORe35r-cw8; zm?HOG=u$78(TGA{y9)o-#rt*M{y{J+Mm=mb0=@7)KM38Xl6xXftVbFWyJ{Re!?@K2 zd<6@LXV}5LB#w-^s{@9?@8ocaiP3Fo4BWx*0=Vw8W6$GGRiAR?Js$T+mLI#J6KCfW z?)%i%M8%IEfb06xU!eW+WoB$+%I_*MUnBH&DlK?#Vd^(F?QR zI>Ds(*i|rS>3aSCruP#orF&oe8l&7Tx`PD~SYH)K%PsuZ@)8)~z|C>as{SA0ZQ#=% zr4mroq1{dIQb=d_8=@+zZpFRZWp)BBCHMv+rFTh!3|Ek@GIr;+Je0N=MuD0cxO0+$ z>p7t;*$jS2?qUenfcl zALx(59pU!OLODQUCXuDxI9R?rH99SDE5N9yYJ>^9@z3;G*DM}qX4-rgD28+vH1ACR zP>p$y3>_HrLg#TmZKg)kZ1TU3oneLzRzCwSdjbWY?>NSf&r<(#Y6o@eYV~ACrZP2u zAV03(u2p0TO$R;-i6F~V*3pj7!b9`PPq)?4iNOi~k;!#m#BcgxJ^8~1U-!KHvAmuN|o?Xc9H5PZJF;?U+VH1Jg>y-iZpgEf;*!&Fu##h;G2LJ_+ZQ@XmQs=Ww*!1rLoVD-H)S17nmoK3petdh@B*4GULu3v!W}ctZ#ktAVu) zHRhsbg^Ck7l$okkRs=JzFR#8&G9n|K$jN`fi3~pQIhQP4LjA(FuWVGn}P7(et6}fnvd6V)aq+5h5EH*`L^-p;Aj`*9d&%T zD~!j-@B7-FI`uqvE9JJ^v5~gl2!3_Fb?io6^8A*u7oudXknW>+xW68_+%vkym&bTv zBnvbEoA?bz!NeEX2wbZ6jmlVa?5`)0pj2!6n6Mk0W6&bn?wLGm(O9$aGhd#2``fI- z(S;FS`YNXqQUrbG*A7v#u!-`8rk?ogi5$y zu%~JXE(uvJLyi^L!wlzl3nn<+s=4#JOsi90K{5H4L_vdEN|@TwX!b8{uCw)efMUabNDDC4{hP*_O+WfwLh z$LWO(sT1dqG~Os8W3@E+cuOc|t`!Zngl_x}FT$Kz9^i>)t!H}fL>2u``dYHm7^#g> z(b2#|k&p1(NulV(m8N=zwCP5AF(v(lW|b{fd10l6E^JNiZpHi6I1|_=sye{2Pi?N9 zdPIn*F2FAYfsI;6)$_Xauqa~ZS7qfd*&$mehs@h8%&9lx@0>jZ;HMK-H7T;aIK1vY zFVOSO%hJ6uSbA6u=mw@6^bG0LihjW;nm3J>6HQs=Qeeej=tZb08LWhu56p(+$R(K!4{x`@ zH0!44;XDK1SCN^`3}{2sk25_jQl2DV;t2z#);C8DQE+5ZUI%7UfPY1xRcp}%{OH8|aTncph+}QvVf*1Xcn_j#k?L?rXmO9G)3XEqK42#gagd1>qI+7DxM9)F!9#){ed(Mi+>L@yTTbaPA&UJr6uF>+bkl1mo>mx;njEYKTF*o5j zv{|1{Bc2*B0N251R4^-|l{1fdk8(Va*m$}l^Dq;sK#_z_LrbfQ|*caWG(Slc75?!A!_PYLW zc3Hn4-|=t87tVl{mAu#jc#fLl3Bbw|`?Mi7Rq1|Lhxy;`iK3p;nl1GaG;M7@rVO>D zI^IjJR{<>Jn1A@YNl<&bddz8axE;m&l|$;%k)F!|B)|(1$kN=V*ZSwRN~siFlsmN9 z#f=vR|8EOaE6U>h_`B1I)+_EB-@%I7nY-Zge3RhQpW?R}YvGB?GzM04wlB{C?G)c| zl+2P;p;%(R9jB}v;{=96V+R11xRZ{v-CHd{|@?$@(Y+I-RelJ#)4_+P+XHsGP&I7 zUT#&AFhq$?O56Yd?0R;xwA)x`7ROkStHW{(L`U zrNNyR7!WGnAbD8r)q(3(NcZL3v8K7Lg*-ypDS-2gyv}RfcN+(rM{x^LYWUH>!|+0s zR;QXS!lPk7a6u(y-E;l-YmL0!isY`J6CEw}WfklMD{8D6%ve3e_&~Ir5VKQF-Tfya$L=`(s+ zesiKGL&n_nv@ChSD1BPF2IJWN?5FdkLil+{l|)&8ve;{tTH1S*K$ribu6^Bo&1JZ)j)`uRUiUA7!hfiW z9GjUbhWw&prWi((R4q@e@t0tf{h?rOcGz zj`@+1k*hz&Ndv6NLvmrITC4FX9xTaM_{6oJK5t$Ma?knRPeaLH!vy%>DF(LE{ zN15LWH!@1(4#hUmJkFyGisIccHPgj5cz;3)-ky04$;E%~OptZx9CjTibOc29U0z}2 zT^6g=6z-)j7%4p_eXg*#crNBWgl}r9TW3$!cEEFT;|5!n`bL)y3x*2MUM+ann z6l!wJOpkOcr9b$$BZ2b7)$d{GPX_Y!K1 zLrT-pMjPj=1&kl}5EnU>u;b;jxBl64|JI;4HT}h~KQtiLqE`hS@86_q=h6f}D2f$D ze?|SOL_p1cO~og{q`B|6#6|^bn|g8(oPZ_iZ|W+URr~gxeDbcEOhl@pusR2VZHg94 zL4oAqY^4~>k-#A2vinew@9hq<2>P7|ZK{>ubScaCU~ilc<<4zCqlz*w_p-bIxA<6a zYxsGkQBm`|NW;FILDRNfE-za+Z!A_#^Ik_ei$2YmeJ!E&N-O=GV_AvSy4`i@>HU~O zSOp-5aWgKqPHHkRduXRXXA%QMUozd0vCF5cB&U^UF&&CdBoKrgok*nW|3E$3|EI4p zX`n{n@Fq;xb8a;Y^=?z}%;@T^mW4BCf_XSCgD~(tu(#Qi=Tu5uE{bEQ{3yH3_KO69 zpnL+iI&Y^S(34@)3kj%*T{fvTGq;qK-y^Kk8s2wF_62AnNfb#3+~-)7>U^9U-jUCy zhsz;KkelEY2QnLEQ$ai#2rGS61y~za*7>$f!8i+Ry6M~i_bctOq<$BP|Lo;hUK04S zW!Ukw82SNnaqw#B)_D{EO8ZIn6;y|mAWfWRcTe3*Dv9{UyGv*q1-aU~&5P;)I?0x( zs%^LJ^E6~+_QfMQinhgzDMCcp_ER!6OQsiDUz|*mnTJQJ zaq56JF@!udJcHlqCx+8|DXpYaH=0v-$>6g%s>dcMakAwRXH>tdTF5)}`?ulcv4)J2 z?N)wrMo@&QN@iH%$ubTjD9Uk#75N|6Cv5~pBIdlEro(Cvbef)$#~NjGHi=Gpp+Rj3 zMET-L7ov<$F{6@M>X@Ykqvk$uaKaj_=DL#Pw0QhL>HLB`+v5qR&10xe<9&m;GwZ%6)4 zzGobyU4au%jW?&YB&6-1vxOtp+l%+Hr;&I^WXqZkfAU!3;}iS7<*jJnKno{-89+$= zs+0bWy^FkSnT|(-P6r1XxoypjgW#S3udsZ9uw0|cMEoL4&1gX2J7R>2q$+|0V-l5X zM6&+2wJpK~&kCf560gysH6y4dQ*sN0K{m~#Xp0Bu5KSn9^NK>j=3P@mPmzFN18QVt zXGacK(@EF0k%n?j{=I;hAEDnC7u=@L(-Rb!N_AkrR5y&0q15v@*!FI;3#eNB4uv;U z7)TDsU%zBu@G08Hh{-^!m9IKrtqV7$AGYLh1IW>%GPMg#K`4%vw3t)-J6vkNv;#J4 zhM20f@3Ye{@GFA8>bV2svAF0OIi;}R3?`_NO*02_#Vl-N7+0~U#q;saCN%Q57ZO4W zO#~tLKLIdwUr^aJs)0BPZj+6>Niz;DD)ebjc)!$I(3HQ?LQeRe7Ysah;JkW^d=6Bl zKR%Kth*Qn8GZz^A@EfLvNI#zL5rqz{*zct<`qO9$#(5rbxkiG*Aw0&tU$C>zk*rON z*qLyn>G;;NO0}x`8;A{nPQ+2#)uQQEYoA0Rr|H{FOk{EqlezT~KXzpk=976t4e1fX zkTg*^7sONSYQ>WbCpH$Ih16fqfVH`W7S(P^+z9qIO`Yr}$>>8>KR7{Gl_2K9KYwD{ zk6k*LFsy?in#w+zEY0!d&CN|sA@O(F$8|?WjpkXw12_=K zZG08#^S+jgnF@5BBov`X^~*S>J0cCjl4ME>|00_bt**8jDEY-QCObyIpvS5gl- z`j=bn1%KdWMPn37+*T|u(fTg*BdFRauz{Dk-oQdF^%27AyZz9;;shgSdYZzr=M_S& z4p-u!jPllKC)y_Lra6Bz%aG_}_uW&#VWipNd%zb}aI44lkL%jHJUyF!CcXjHv_HFz z?aRJufK)u}GO{d^R6;1rIr?)^VN<(yeU%hx*8v5zE#% zo*r9|%-5+zKX}lrg@8?fxdXZ2#HU`%ELf$`|Ak`eLp)MCVFjVTVuf#fa-yG8F|&wI z?wgp$1L9v^h&i@zIA4FOwl+Lo%|P^hPMZd9TX~GP6yferX&N5gHslydDo?TB-Q8Dy ziuhH+t~PG9{Ig5IjC^htq$Gr+t+LRK67qjAv$K<@s~})i!UR}Wo^#5h;LV5Zc95m| z*SS~Gknd$$)=NIH5b&}tRM4(xWvh8Y{~Sd^(AEi(FadIRUD>8g4S!xl*{v1!h0+cH z{@w}>$w9U4^))iCxeTJ<5#tY=eD)EA%q^j6O9bTCZMa}(-?~3N>`rYUZRX6G213H~ zBk(cGI7Br=^4GCMn^<^Ax`6NjhtT8W`C$r_CAI*IA`1xxud;7kKZm;XH?9T35C*?g z0W>^M_IF}UGv$S2>k_|uegGoXaf@Hk7!r5PS=szE4!x*xbdOFzIjIJQ zX**bNIwmIjPrS7qZbA;DBd;5;Brv8i2|V2&GIe_0CwGB&T$lfRD6Mca>a84LQM*_p z9p{>i=g9-T3}iR#)q~KBRXt5t9`KD0525!POa@bEs5gFWrt$5c&fuAgZax20=`1Tbyy~B=hzf0ok zDLFdEx4QJ|Qv*0fI_JdISy)!&wuSO~jvCE|S%W?o+ltkg%`GfoFflQoc7yMuGqd@L z&Ni+cc&z#4>H^levr{HE1VX94*$aYyI-`>!`2{5!l- zR_V`lN@ocmMdl`IXcHA^Fx2S$+(NtBOVMW);F)?pa=W8$8;GbcHgMO}HMTIPCL!lv zy9vlc#6YBIWSY9rNEp}yW_!Z2sFngbZT6<5+~|c)4;HPnelNx5I906=oW!w=9Q|S4 z@I77qMT00EtxXnN{BRuH#$({V5(QdPG&A!-*w~@yxo#@^pbgO;tYKL)4K#)K6Tq?y(0|d4su%oSLAEM;YWd77 z5e{9hS;dMBy)jD>))&2bO!T=1-j?&(ocKCTSljEje>6>YW=AZtcEc#2?t5rqe9v|S zn~5Ym{m9Y+l{RC{W=%U6W}A&r3^OX!lO_G_8w@7xcqX85axntJ>95%6B2npZ7z#OJU@GVyQH#yWVESpBV zH=0DO02U!t83^Frz?bC78#sz8Y}+$3XR^v)Hdi^)Sa3Rbf-n9&z4xu&>u-R|fHP&` zfxoY&0qfa6Tyrz?21P#Kb6oHD=q`M1+zCoodP6R4tw)eLPZQM@YJPu{(Kx(MOtD-B zx|8Zn;2ip{as#X=OLSvbvUY7ING0g!^<$AJV|ArV8tvf1)xgQHu&xd_hg%4B>J}o5 zdFr;vBtnl5o0^|x-JO(kLy@B+$Fhv!Ft(3dNuhOP_*K4B{!tQ-=W17CN&lDg5wg9t zM0vlOJVqo&t{P%sclX}f0{h75Z-T80XKWQfl6t{$$RotVWlbf3nQYcn*x1!GN)uO<`#-}QgNFqpHk+w z2X>|bXL4vE*s&%9D1&UMq?ggEz{_%7)He!}7_W`Xu|5F|#NJ*gTIvnkTC2Ee4+Go( z@tdoNF=*>p1{W^V!Tf&ta>HAc2m$+;PAOv)A%;l5&prb0XY_h35;V2M)AQ1ykTfyA zGuBe&dN+;Lb5~P90@EM7xcxTE%3Q*;s+iC-vvG4usEOMhD0H~bH9pB=dh#5ON1|J3 zG`KTBp>>x!y-F939=0Bxv6X7jGcNw4p8Km<+WrZnDIU3!dM>Xk6cQ2=LICMJD=W}W z4TiRZb0hZlN-;(nFD#S>+Z;h`G8OZmbMhC472o8?{Dv-p0&Qpj|7Dr@xZ2HHXaHYS z19IVIc&LI3NWSlJHKsVmf20i2>fm5zwY+#I-E~-sGE-+G_*&FCDG%`NgqqNM^V9Kd zlbGd=<0=23?KHmjssf&y^sWEmUH!;oGEhr78AibCOsYn0U`wTE*=+Us!4iG=PdPQ} zfiGMXT|*XK4I;S^VEw8Vq~~@8csOpAV6vd+IKPnW`qF%#!vY|9$Z0b1q3%%idoCG;XJz46>$laC zO=fmxWXH16T4W73gk}rckGd{%38zqdBwt0Qn`N=wuUeM&S3wGQa74yXgV;;tC^TG# zrS(U^OW#JMSo`o8R}b9%Tql!?5nPFi*PXTnHb1v9jbB#%c|Oj)gJgUbLAp*< z8g1nP&f7x=KPBuVbs1cUDgy+70k&4+5^^D4N`qjhUiuG882}qEZ%BW?*xmg-wubp0 z^14Ppsm!mF4)=t1jpxC?gmFX-mbPE!8ow+mNd}^YT&c%tXdBgzSTU+gqq)6}4EZPt zlbYizg0zU=s1h+`FpB5~c5@1@df(DI+bEniBdG$8jt4o6J1%LHxD-@x0gF3O&$q86 zJ(!{1ZK&qpL&y+8$vtrla<&Np-+?@+_bd#L%vIeED?+hm;4)d@1euBujWXTx^*UwV?r3C9O zeB-WOvLmuYaC(apxO^Gg=`W@gPNG_)cAj;`bMU(Z?(UO{G&xh4o^ewZ)(z_XN8MDc zM802kJbPcPGvBMrqbL&S4E5~n1wMfU;tgv9G;(qORoBTYoif(gC3VHbHS;!?aePchuq}&dxW+# zKD;bsIRoBpH%3EN2}AxT6&vHA7~WG+FuJ*|Euw(;dH$ky8S0&}NWc{QRoQC(;8JVg z_DApY)w#3JeF7o#aA5@{@6{VQ^=}hhIK^-f?P}|=?rNlf*aRFzn4K{Sgi5UjE5DbQ z=`3JLl4w-N=EdZ97bcU>)?M>3MyVh)8TO%f%p@PU>7sDJo=MK^Y^4c4o~_!j69G%{HQ%6hXI7J zQ1B3&Y=kffTII>DEbIKqkM)9&jAtq(?~^#Sf*Iq<$u}a#*He8jl7IC<6Ms&yCEm-I z=xyd1zzK+G6^1&ba30r)Ak7XorAoWsHMc4KVJ^B10|rvQHS!}n6bSTX4gYj-y=0*t@RYkYWZd@@I>d zLUGkfhtsiJyN_%Ek&Ua4*?>l7=Jb6jnKbpLrkG-c@NnoOkAV2ZL@=L=EpHD9o8)BV z@}i6&OBAQ3s=%=j{W+9dLF1T&NRGho*t24VCz~R<%!3qXWlE4$YP&?@MLFq;i2*GN4H~zXX%xfrt!Q+X z`OU5K&x?77?<%@o=-RY!7NqFI628{P<;QF_Ga;#Bk$*Q6 zY_-LV(ncQiG(^681S7wVe3Av+EnNXf5T16P<+Cu5aXW=~5_qdvJB%Z=!fs((lx|8k z`LH&eHnmwyH-h{5n$gyEKS5acxkUOtWR2m&@nYuh`+MRjNVEN(9KwF6cR_PhN4+(n z1u4&GpM8n`agpebXRN%PdyE=sp)tJj^!MJLhlWTDTc8`H@)=)iRa=|${j_MoAf8B@ zf?c_)4n5@WF+WfY78d`9X6m#76m$S7Y9K|#eY9^69Y>}p&Pz;?jXQtun)Yc z+O%!$A@m-C8Qz67fMpRkb@Hfg-XZvFpBzLIlqd*Z58~r2+?gY&)zQwU0vlPd=W`Fj zyd7cf&6%jL6k78&d0%LszLaVu$Yxs=JpjLNBNVcO!f+u|12b%h+oT= z^P1fU5Soi)%RCKz>yJwf7IJ7a(QT#Q|5nNyU_lW4Sp%ORd3g@=d(?)%_)|+8IvfJ( zDIdP=JJIUfN18qDrUTB~ID5lTvL$S!mP=a(mE0C@;!aALZRO;N6tS1@+|SOvR5dEO zSPjmh9p3&%Y2t(tc7qNtCMV#&V;E)MlGLl8zZilTP)`Eysq1e#O>B=fk^MNv*^KtYcCZo3^&L)OfWFxAs6JaP4P@jev^WY2*b7O zoopN&;dBo?2dUqr`x!rW}gVdRJ?k)iPbu+FxSugVjqPEL@) z?+d-J87A9Wffz@ky`qhN;ZSEHhJyTe*U~lrN)ZO9nU={iWwto`M!d&A%~CS^t~Xaw(Bb3p zO!%U#Z^-0zXe$<(hJJxNNoXXB7lkmWQa`TNpqUcyvMQ9Ej z_2NxTkS`5ms56+t)frz)M&xvU_*1!Eh9-0dANu*{;ap&iwoSN&CZ6(UJ~7b!_Z>+f z`H|?SBdVL7{z1}GHIeROxu&UZS66yzGxFD-pUEzi(1Yl&2Z91Lwy}wq;RJ$693vbZ zR131}ZCahfY5Eb3NaU$=9#R$gB|q%OuTgt`XjQ?24%F0J{D|`p!Ai9SezJb}k(7Ajb)e!ia@@FAJxVYgd7h zcxtU$Tz;=&p=~%e@$kVz3wrc_nwcW~WJCadHhTvU^{GA?m597)!*MRX8;~JFg7_e} z3XmV>(;qqQiZGc2|JZKyTV>toQ^NN6<4M;uA~Ax{BXXO+jV@Qi9gS-L0mo&X-^w13m? zm=jc^$E3|**+NaD>w+#F@C5<_2JOEc0c+<}Qd{sY@~8u>l3!`TvqM#h%)J`$X?4@p z#L>9kuh6#rlWKH@XNP6{%!Ec`7JVrwK=^B0llbyaLnAYi2Nx)@nd$%Auh1a5RSmn!F(D zQm976h4_d-@XB!R_oji6@LMY*XbYY^nREE*j~h4zn$s%xH!#Jc)QJ*}A`GIRd%;h9 zr{!bY3}7`s!-Rl>klFG##BXj3?n6$#d%tgdy|x2#n(E^$MU+YRI8Z8*;o*gmMTmW) zNeBA8zbMS_rYfLnq23plX&v{tAcWdxm}uM4X`YsI`kDOWMkV48n+n*!I1)$_xpm(5 z4=xq4`0u+dvp~6^gnWa*2ys@Chii-{*4xnU^>W+C1J}C)$2 z3hjmJLk_vT=c&wKY<-88OBROPvWDR9e{2jslXygBHpU(P+Zj!e=8Xqum=a=E)D|n^ zdzhnA*VUZnvAIl@E6jsv$oX0mHsWkYa%Kr^@B(U*bR3M;E`>r_wsHdH0r;%BRp%qdj7OagAT4Hfxv~-q3+3s p@Y&Mlv$OgC*Z)S~&FoF&^{D3SR^jz;@J0~`IjQfGwc;jW{{tW!r?CJ4 literal 0 HcmV?d00001 diff --git a/webui/webui.go b/webui/webui.go new file mode 100644 index 00000000..476807ae --- /dev/null +++ b/webui/webui.go @@ -0,0 +1,282 @@ +package webui + +import ( + _ "embed" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/cooperspencer/gickup/types" + "github.com/goccy/go-yaml" + "github.com/rs/zerolog/log" +) + +// BackupStatus is the result of a single repo-to-destination backup attempt. +type BackupStatus string + +const ( + StatusSuccess BackupStatus = "success" + StatusFailed BackupStatus = "failed" +) + +// StatusFromInt converts the integer convention used by backup() (1=success, 0=failed). +func StatusFromInt(i int) BackupStatus { + if i == 1 { + return StatusSuccess + } + return StatusFailed +} + +// BackupEntry records one repo-to-destination backup attempt. +type BackupEntry struct { + Timestamp time.Time `json:"timestamp"` + RepoName string `json:"repo_name"` + RepoURL string `json:"repo_url"` + Owner string `json:"owner"` + Hoster string `json:"hoster"` + DestType string `json:"dest_type"` + DestAddr string `json:"dest_addr"` + Status BackupStatus `json:"status"` + DurationMs int64 `json:"duration_ms"` +} + +// ConfigInfo describes one loaded configuration block for the UI. +type ConfigInfo struct { + Index int `json:"index"` + Name string `json:"name"` + Sources int `json:"sources"` + Dests int `json:"dests"` + CronSpec string `json:"cron_spec,omitempty"` + NextRun string `json:"next_run,omitempty"` // RFC3339 +} + +// store holds backup entries, config file paths, and the run callback. +type store struct { + mu sync.RWMutex + entries []BackupEntry + configFiles []string + configs []ConfigInfo + runFunc func(int) + running int32 // atomic: 1 = a run is in progress +} + +// Global is the shared store used across the application. +var Global = &store{} + +// SetConfigFiles registers the config file paths with the store. +func (s *store) SetConfigFiles(files []string) { + s.mu.Lock() + defer s.mu.Unlock() + s.configFiles = make([]string, len(files)) + copy(s.configFiles, files) +} + +// SetConfigs registers the list of config summaries shown in the UI. +func (s *store) SetConfigs(infos []ConfigInfo) { + s.mu.Lock() + defer s.mu.Unlock() + s.configs = make([]ConfigInfo, len(infos)) + copy(s.configs, infos) +} + +// SetRunFunc registers the callback that main.go uses to trigger a backup run. +// index == -1 means run all configs; otherwise run the config at that index. +func (s *store) SetRunFunc(fn func(int)) { + s.mu.Lock() + defer s.mu.Unlock() + s.runFunc = fn +} + +// Record appends a backup entry to the store. +func (s *store) Record(e BackupEntry) { + s.mu.Lock() + defer s.mu.Unlock() + s.entries = append(s.entries, e) +} + +// Clear removes all recorded backup entries. +func (s *store) Clear() { + s.mu.Lock() + defer s.mu.Unlock() + s.entries = nil +} + +//go:embed ui/index.html +var indexHTML []byte + +//go:embed ui/logo.png +var logoPNG []byte + +// Serve starts the web UI HTTP server on addr. +func Serve(addr string) { + mux := http.NewServeMux() + mux.HandleFunc("/", handleIndex) + mux.HandleFunc("/logo.png", handleLogo) + mux.HandleFunc("/api/status", handleStatus) + mux.HandleFunc("/api/config", handleConfig) + mux.HandleFunc("/api/configs", handleConfigs) + mux.HandleFunc("/api/run", handleRun) + mux.HandleFunc("/api/running", handleRunning) + log.Info().Str("addr", addr).Msg("Web UI listening") + if err := http.ListenAndServe(addr, mux); err != nil { + log.Error().Err(err).Msg("Web UI server error") + } +} + +func handleIndex(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write(indexHTML) +} + +func handleLogo(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(logoPNG) +} + +func handleStatus(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + Global.Clear() + w.WriteHeader(http.StatusNoContent) + return + } + Global.mu.RLock() + entries := make([]BackupEntry, len(Global.entries)) + copy(entries, Global.entries) + Global.mu.RUnlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(entries) +} + +// handleConfigs returns the list of loaded configuration blocks. +func handleConfigs(w http.ResponseWriter, r *http.Request) { + Global.mu.RLock() + cfgs := make([]ConfigInfo, len(Global.configs)) + copy(cfgs, Global.configs) + Global.mu.RUnlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(cfgs) +} + +// handleRunning reports whether a backup run is currently in progress. +func handleRunning(w http.ResponseWriter, r *http.Request) { + running := atomic.LoadInt32(&Global.running) != 0 + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]bool{"running": running}) +} + +// handleRun triggers an immediate backup run. +// POST /api/run body: {"index": -1} (-1 = all, ≥0 = specific config) +func handleRun(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if !atomic.CompareAndSwapInt32(&Global.running, 0, 1) { + http.Error(w, "backup already running", http.StatusConflict) + return + } + + var req struct { + Index int `json:"index"` + } + req.Index = -1 + _ = json.NewDecoder(r.Body).Decode(&req) + + Global.mu.RLock() + fn := Global.runFunc + Global.mu.RUnlock() + + if fn == nil { + atomic.StoreInt32(&Global.running, 0) + http.Error(w, "not ready", http.StatusServiceUnavailable) + return + } + + go func() { + defer atomic.StoreInt32(&Global.running, 0) + fn(req.Index) + }() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]bool{"running": true}) +} + +func handleConfig(w http.ResponseWriter, r *http.Request) { + Global.mu.RLock() + files := make([]string, len(Global.configFiles)) + copy(files, Global.configFiles) + Global.mu.RUnlock() + + if len(files) == 0 { + http.Error(w, "no config files registered", http.StatusNotFound) + return + } + + q := r.URL.Query() + + // List mode: GET /api/config?list=1 + if r.Method == http.MethodGet && q.Get("list") == "1" { + type fileInfo struct { + Index int `json:"index"` + Name string `json:"name"` + Path string `json:"path"` + } + list := make([]fileInfo, len(files)) + for i, f := range files { + list[i] = fileInfo{Index: i, Name: filepath.Base(f), Path: f} + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(list) + return + } + + // Parse file index (default 0). + idx := 0 + if s := q.Get("file"); s != "" { + n, err := strconv.Atoi(s) + if err != nil || n < 0 || n >= len(files) { + http.Error(w, "invalid file index", http.StatusBadRequest) + return + } + idx = n + } + + switch r.Method { + case http.MethodGet: + data, err := os.ReadFile(files[idx]) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + _, _ = w.Write(data) + + case http.MethodPost: + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // Validate that the submitted content parses as a known config shape. + var conf types.Conf + if err := yaml.Unmarshal(body, &conf); err != nil { + http.Error(w, "invalid YAML: "+err.Error(), http.StatusBadRequest) + return + } + if err := os.WriteFile(files[idx], body, 0o600); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} From 3369a9fa37f88b0aaa31fd8673faa5590fa27c5a Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Mon, 4 May 2026 12:15:38 +0200 Subject: [PATCH 02/44] fix gitlab format issue (#386) --- gitlab/gitlab.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index b35518b6..270243ae 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -562,7 +562,7 @@ func GetIssues(repo *gitlab.Project, client *gitlab.Client, conf types.GenRepo) } else { if len(i) > 0 { for _, issue := range i { - issues[strconv.FormatInt(issue.IID, 64)] = issue + issues[strconv.FormatInt(issue.IID, 10)] = issue } } else { break From 17c2361e3da16ae29284c5e5002fdb7cbe808b87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 18:14:58 +0200 Subject: [PATCH 03/44] Bump github.com/go-git/go-git/v5 from 5.18.0 to 5.19.0 (#387) --- go.mod | 6 +++--- go.sum | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index f65a4d6e..b5034bda 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/alecthomas/kong v1.15.0 github.com/cooperspencer/onedev v0.0.0-20240801121041-919f6d3f7ea6 - github.com/go-git/go-git/v5 v5.18.0 + github.com/go-git/go-git/v5 v5.19.0 github.com/goccy/go-yaml v1.19.2 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/google/go-cmp v0.7.0 @@ -46,7 +46,7 @@ require ( github.com/emirpasic/gods v1.18.1 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.8.0 // indirect + github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-ini/ini v1.67.0 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect @@ -69,7 +69,7 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/philhofer/fwd v1.2.0 // indirect - github.com/pjbgf/sha1cd v0.5.0 // indirect + github.com/pjbgf/sha1cd v0.6.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/sftp v1.13.10 // indirect diff --git a/go.sum b/go.sum index 863db697..3c100d96 100644 --- a/go.sum +++ b/go.sum @@ -66,12 +66,12 @@ github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.8.0 h1:I8hjc3LbBlXTtVuFNJuwYuMiHvQJDq1AT6u4DwDzZG0= -github.com/go-git/go-billy/v5 v5.8.0/go.mod h1:RpvI/rw4Vr5QA+Z60c6d6LXH0rYJo0uD5SqfmrrheCY= +github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmmBPA= +github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.18.0 h1:O831KI+0PR51hM2kep6T8k+w0/LIAD490gvqMCvL5hM= -github.com/go-git/go-git/v5 v5.18.0/go.mod h1:pW/VmeqkanRFqR6AljLcs7EA7FbZaN5MQqO7oZADXpo= +github.com/go-git/go-git/v5 v5.19.0 h1:+WkVUQZSy/F1Gb13udrMKjIM2PrzsNfDKFSfo5tkMtc= +github.com/go-git/go-git/v5 v5.19.0/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= @@ -155,8 +155,8 @@ github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= -github.com/pjbgf/sha1cd v0.5.0 h1:a+UkboSi1znleCDUNT3M5YxjOnN1fz2FhN48FlwCxs0= -github.com/pjbgf/sha1cd v0.5.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= +github.com/pjbgf/sha1cd v0.6.0 h1:3WJ8Wz8gvDz29quX1OcEmkAlUg9diU4GxJHqs0/XiwU= +github.com/pjbgf/sha1cd v0.6.0/go.mod h1:lhpGlyHLpQZoxMv8HcgXvZEhcGs0PG/vsZnEJ7H0iCM= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -215,8 +215,8 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= -golang.org/x/exp v0.0.0-20250813145105-42675adae3e6 h1:SbTAbRFnd5kjQXbczszQ0hdk3ctwYf3qBNH9jIsGclE= -golang.org/x/exp v0.0.0-20250813145105-42675adae3e6/go.mod h1:4QTo5u+SEIbbKW1RacMZq1YEfOBqeXa19JeshGi+zc4= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= From 5f03e6cfa6e2eb01f218ca4529891c6157879d27 Mon Sep 17 00:00:00 2001 From: Hildebrand Soethoudt <54576297+Hildebrand-Ritense@users.noreply.github.com> Date: Wed, 13 May 2026 10:04:32 +0200 Subject: [PATCH 04/44] updated ktrysmt/go-bitbucket dependency to v0.9.100, that includes the BitBucket API deprecation fix as part of release v0.9.96 (#389) --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b5034bda..bbcb2d3b 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/go-github/v74 v74.0.0 github.com/gookit/color v1.6.0 github.com/hasura/go-graphql-client v0.16.0 - github.com/ktrysmt/go-bitbucket v0.9.96 + github.com/ktrysmt/go-bitbucket v0.9.100 github.com/melbahja/goph v1.5.0 github.com/minio/minio-go/v7 v7.0.100 github.com/prometheus/client_golang v1.23.2 diff --git a/go.sum b/go.sum index 3c100d96..5bc2fe71 100644 --- a/go.sum +++ b/go.sum @@ -133,6 +133,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ktrysmt/go-bitbucket v0.9.96 h1:KI+ePerKCIJHkZNFV/fJkk/8yfWTQDgr139UOIp/RJY= github.com/ktrysmt/go-bitbucket v0.9.96/go.mod h1:/1H7KMC7FsdkRoNMq21an6dyFpA+irNw5tyYpHzRC2Q= +github.com/ktrysmt/go-bitbucket v0.9.100 h1:d0CWilBnJ2ywomeH0/1QNULsEUziye/AD0cOEKfCMx4= +github.com/ktrysmt/go-bitbucket v0.9.100/go.mod h1:/1H7KMC7FsdkRoNMq21an6dyFpA+irNw5tyYpHzRC2Q= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= From d3fd3e4717825d9d731cb5e5fae2d19c519600e9 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Wed, 13 May 2026 10:10:42 +0200 Subject: [PATCH 05/44] go mod tidy --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 5bc2fe71..232d8afa 100644 --- a/go.sum +++ b/go.sum @@ -131,8 +131,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/ktrysmt/go-bitbucket v0.9.96 h1:KI+ePerKCIJHkZNFV/fJkk/8yfWTQDgr139UOIp/RJY= -github.com/ktrysmt/go-bitbucket v0.9.96/go.mod h1:/1H7KMC7FsdkRoNMq21an6dyFpA+irNw5tyYpHzRC2Q= github.com/ktrysmt/go-bitbucket v0.9.100 h1:d0CWilBnJ2ywomeH0/1QNULsEUziye/AD0cOEKfCMx4= github.com/ktrysmt/go-bitbucket v0.9.100/go.mod h1:/1H7KMC7FsdkRoNMq21an6dyFpA+irNw5tyYpHzRC2Q= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= From 252377824199042da11780219625762adf4cc2e9 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Thu, 14 May 2026 18:19:58 +0200 Subject: [PATCH 06/44] show correct next cron --- webui/webui.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/webui/webui.go b/webui/webui.go index 476807ae..e011b8d7 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -159,6 +159,20 @@ func handleConfigs(w http.ResponseWriter, r *http.Request) { cfgs := make([]ConfigInfo, len(Global.configs)) copy(cfgs, Global.configs) Global.mu.RUnlock() + + now := time.Now() + for i := range cfgs { + if cfgs[i].CronSpec == "" { + continue + } + sched, err := types.ParseCronSpec(cfgs[i].CronSpec) + if err != nil { + cfgs[i].NextRun = "" + continue + } + cfgs[i].NextRun = sched.Next(now).Format(time.RFC3339) + } + w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(cfgs) } From 97c1d192bd53bd6c146112429d15897bd1676352 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Thu, 14 May 2026 19:11:11 +0200 Subject: [PATCH 07/44] fixed exclude org race condition (#393) --- bitbucket/bitbucket.go | 8 ++++---- gitea/gitea.go | 18 ++++++++++-------- github/github.go | 16 ++++++++-------- gitlab/gitlab.go | 22 ++++++++++++---------- gogs/gogs.go | 17 +++++++++-------- onedev/onedev.go | 7 ++++--- sourcehut/sourcehut.go | 8 ++++---- 7 files changed, 51 insertions(+), 45 deletions(-) diff --git a/bitbucket/bitbucket.go b/bitbucket/bitbucket.go index 24a5ff76..796f6d71 100644 --- a/bitbucket/bitbucket.go +++ b/bitbucket/bitbucket.go @@ -125,6 +125,10 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -141,10 +145,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, diff --git a/gitea/gitea.go b/gitea/gitea.go index 620c519a..19a19338 100644 --- a/gitea/gitea.go +++ b/gitea/gitea.go @@ -322,6 +322,11 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { continue } + + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -354,10 +359,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(repo.Include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, @@ -468,6 +469,11 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { continue } + + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -500,10 +506,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(repo.Include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, diff --git a/github/github.go b/github/github.go index b7c4d9e6..581e1e49 100644 --- a/github/github.go +++ b/github/github.go @@ -264,6 +264,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } + if exclude[*r.Name] { + continue + } + + if excludeorgs[r.GetOwner().GetLogin()] { + continue + } + if include[*r.Name] { repos = append(repos, types.Repo{ Name: r.GetName(), @@ -286,14 +294,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[*r.Name] { - continue - } - - if excludeorgs[r.GetOwner().GetLogin()] { - continue - } - if len(repo.Include) == 0 { if len(includeorgs) > 0 { if includeorgs[r.GetOwner().GetLogin()] { diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index 270243ae..e74c7d0f 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -269,6 +269,12 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if time.Since(*r.LastActivityAt) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { continue } + if exclude[r.Name] { + continue + } + if excludeorgs[r.Namespace.FullPath] { + continue + } if include[r.Name] { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { repos = append(repos, types.Repo{ @@ -305,9 +311,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } if len(include) == 0 { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { repos = append(repos, types.Repo{ @@ -436,6 +439,12 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } + if exclude[r.Name] { + continue + } + if excludeorgs[r.Namespace.FullPath] { + continue + } if include[r.Name] { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { repos = append(repos, types.Repo{ @@ -472,13 +481,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if excludeorgs[r.Namespace.FullPath] { - continue - } - if len(include) == 0 { if len(includeorgs) == 0 || includeorgs[r.Namespace.FullPath] { if r.RepositoryAccessLevel != gitlab.DisabledAccessControl { diff --git a/gogs/gogs.go b/gogs/gogs.go index 415f2c80..360f7e49 100644 --- a/gogs/gogs.go +++ b/gogs/gogs.go @@ -182,6 +182,11 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { continue } + + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -214,10 +219,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, @@ -309,6 +310,10 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -341,10 +346,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(repo.Include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, diff --git a/onedev/onedev.go b/onedev/onedev.go index 098c7b70..1632aa95 100644 --- a/onedev/onedev.go +++ b/onedev/onedev.go @@ -95,13 +95,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } } + if exclude[r.Name] { + continue + } + if len(repo.Include) > 0 { if !include[r.Name] { continue } - if exclude[r.Name] { - continue - } } defaultbranch, _, err := client.GetDefaultBranch(r.ID) diff --git a/sourcehut/sourcehut.go b/sourcehut/sourcehut.go index ac0ef07c..e40ce628 100644 --- a/sourcehut/sourcehut.go +++ b/sourcehut/sourcehut.go @@ -306,6 +306,10 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { ownerName := strings.TrimPrefix(ownerCanonicalName, "~") isPrivate := strings.EqualFold(r.Visibility, "PRIVATE") || strings.EqualFold(r.Visibility, "private") + if exclude[r.Name] { + continue + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, @@ -335,10 +339,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if exclude[r.Name] { - continue - } - if len(include) == 0 { repos = append(repos, types.Repo{ Name: r.Name, From 617acfda96b83159c63de7030f3c68a5bdf91ac0 Mon Sep 17 00:00:00 2001 From: Bernat Mut <1116133+monwolf@users.noreply.github.com> Date: Mon, 18 May 2026 11:17:28 +0200 Subject: [PATCH 08/44] chore: bump action versions in workflows to latest stable releases (#395) * chore: bump action versions in workflows to latest stable releases * fix: add revive var-naming exclusion for zip_test.go * fix: correct formatting and structure in .goreleaser.yml * fix: ensure fetch-depth is set for actions/checkout in workflow * fix: update GoReleaser conditions for tag-based releases in workflow --- .github/workflows/CodeQL.yml | 8 ++++---- .github/workflows/docker.yml | 28 ++++++++++++++-------------- .github/workflows/go.yml | 23 +++++++++++++++++++---- .github/workflows/validate-lint.yml | 6 +++--- .golangci.yml | 3 +++ .goreleaser.yml | 20 +++++++++++--------- 6 files changed, 54 insertions(+), 34 deletions(-) diff --git a/.github/workflows/CodeQL.yml b/.github/workflows/CodeQL.yml index 419bc843..b011b5bd 100644 --- a/.github/workflows/CodeQL.yml +++ b/.github/workflows/CodeQL.yml @@ -12,14 +12,14 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v4 with: languages: go - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v4 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 \ No newline at end of file + uses: github/codeql-action/analyze@v4 \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 6c9c1e75..f8d4a917 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,11 +14,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v6 - name: Docker meta id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: | buddyspencer/gickup @@ -33,29 +33,29 @@ jobs: type=raw,value=dev,enable=${{ github.ref == 'refs/heads/main' }} - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Github Packages if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GHCR_PAT }} - name: Build image and push to Docker Hub and GitHub Container Registry - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . tags: ${{ steps.meta.outputs.tags }} @@ -71,11 +71,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v6 - name: Docker meta id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@v6 with: images: | buddyspencer/gickup @@ -89,29 +89,29 @@ jobs: type=raw,value=ubuntu-dev,enable=${{ github.ref == 'refs/heads/main' }} - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to DockerHub if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to Github Packages if: github.event_name != 'pull_request' - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GHCR_PAT }} - name: Build image and push to Docker Hub and GitHub Container Registry - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: . file: Dockerfile.ubuntu diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 7c838159..f7025258 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -12,10 +12,12 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 + with: + fetch-depth: 0 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v6 with: go-version: 1.26 @@ -23,7 +25,8 @@ jobs: run: go test -v ./... - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v2 + if: startsWith(github.ref, 'refs/tags/') + uses: goreleaser/goreleaser-action@v7 with: # either 'goreleaser' (default) or 'goreleaser-pro' distribution: goreleaser @@ -31,9 +34,21 @@ jobs: args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Run GoReleaser + if: ${{ !startsWith(github.ref, 'refs/tags/v') }} + uses: goreleaser/goreleaser-action@v7 + with: + # either 'goreleaser' (default) or 'goreleaser-pro' + distribution: goreleaser + version: latest + args: release --snapshot --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload a Build Artifact - uses: actions/upload-artifact@v4 + if: startsWith(github.ref, 'refs/tags/') + uses: actions/upload-artifact@v7 with: name: gickup path: dist/* diff --git a/.github/workflows/validate-lint.yml b/.github/workflows/validate-lint.yml index fe0815f1..0af49b25 100644 --- a/.github/workflows/validate-lint.yml +++ b/.github/workflows/validate-lint.yml @@ -20,18 +20,18 @@ jobs: steps: - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v2 + uses: actions/setup-go@v6 with: go-version: ${{ env.GO_VERSION }} - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v6 with: path: go/src/github.com/cooperspencer/gickup fetch-depth: 0 - name: Cache Go modules - uses: actions/cache@v4 + uses: actions/cache@v5 with: path: | ~/go/pkg/mod diff --git a/.golangci.yml b/.golangci.yml index 0ccada8d..4841ca90 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -90,6 +90,9 @@ linters: - linters: [revive] path: zip/zip.go text: "var-naming:" + - linters: [revive] + path: zip/zip_test.go + text: "var-naming:" # Per-file existing exclusions - path: main.go text: bad syntax for struct tag pair diff --git a/.goreleaser.yml b/.goreleaser.yml index d865b585..371d7590 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,6 +1,7 @@ +version: 2 + builds: - - - goos: + - goos: - freebsd - windows - darwin @@ -9,19 +10,20 @@ builds: - amd64 - arm - arm64 - - 386 + - "386" goarm: - - 6 - - 7 + - "6" + - "7" ignore: - goos: darwin - goarch: 386 + goarch: "386" - goos: windows goarch: arm archives: - - - format: tar.gz + - formats: + - tar.gz format_overrides: - goos: windows - format: zip + formats: + - zip From 5d3622e30fc4f600dfe3377d1affa3c4cfbfc16d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 16:35:45 +0200 Subject: [PATCH 09/44] Bump github.com/go-git/go-git/v5 from 5.19.0 to 5.19.1 (#397) Bumps [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) from 5.19.0 to 5.19.1. - [Release notes](https://github.com/go-git/go-git/releases) - [Changelog](https://github.com/go-git/go-git/blob/main/HISTORY.md) - [Commits](https://github.com/go-git/go-git/compare/v5.19.0...v5.19.1) --- updated-dependencies: - dependency-name: github.com/go-git/go-git/v5 dependency-version: 5.19.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index bbcb2d3b..dc47db53 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 github.com/alecthomas/kong v1.15.0 github.com/cooperspencer/onedev v0.0.0-20240801121041-919f6d3f7ea6 - github.com/go-git/go-git/v5 v5.19.0 + github.com/go-git/go-git/v5 v5.19.1 github.com/goccy/go-yaml v1.19.2 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/google/go-cmp v0.7.0 diff --git a/go.sum b/go.sum index 232d8afa..74b07b81 100644 --- a/go.sum +++ b/go.sum @@ -70,8 +70,8 @@ github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmm github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.19.0 h1:+WkVUQZSy/F1Gb13udrMKjIM2PrzsNfDKFSfo5tkMtc= -github.com/go-git/go-git/v5 v5.19.0/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= +github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00= +github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= From ee95e89de5329e9471f38e02aee89be63663db64 Mon Sep 17 00:00:00 2001 From: Jan Loewe Date: Tue, 2 Jun 2026 21:00:20 +0200 Subject: [PATCH 10/44] Fix unintentional data loss when zip is false and keep greater than or one (#399) --- local/local.go | 1 + 1 file changed, 1 insertion(+) diff --git a/local/local.go b/local/local.go index 00f1d7d2..94d3f7a3 100644 --- a/local/local.go +++ b/local/local.go @@ -283,6 +283,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { sub.Warn(). Str("repo", repo.Name). Msgf("couldn't parse timestamp! %s", types.Red(file.Name())) + continue } if l.Zip && !strings.HasSuffix(file.Name(), ".zip") { continue From fa7816143d70b3fd0736a99b679003497804e9a4 Mon Sep 17 00:00:00 2001 From: Bernat Mut <1116133+monwolf@users.noreply.github.com> Date: Tue, 2 Jun 2026 21:10:28 +0200 Subject: [PATCH 11/44] feat: Add support for S3 AWS Credentials Chain (#394) * feat: Add support for S3 static credentials configuration - Updated S3Repo struct to include UseStaticCreds field - Refactored credential retrieval logic in UploadDirToS3 and DeleteObjectsNotInRepo functions - Added tests for S3Repo and credential retrieval functions * docs: add changes for use_static_creds * fix: lint issues * fix: exclude gosec sanitized file path --- .golangci.yml | 2 + conf.example.yml | 1 + gickup_spec.json | 4 ++ main.go | 26 ++++---- main_test.go | 157 ++++++++++++++++++++++++++++++++++++++++++-- s3/s3.go | 12 +++- s3/s3_test.go | 49 ++++++++++++++ types/types.go | 1 + types/types_test.go | 18 +++++ 9 files changed, 252 insertions(+), 18 deletions(-) create mode 100644 s3/s3_test.go diff --git a/.golangci.yml b/.golangci.yml index 4841ca90..5ea7704e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -62,6 +62,8 @@ linters: text: "G704:" - linters: [gosec] text: "G117:" + - linters: [gosec] + text: "G703:" # Global exclusions for staticcheck style suggestions - linters: [staticcheck] text: "SA1019:" diff --git a/conf.example.yml b/conf.example.yml index 274ec366..1a159b45 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -275,6 +275,7 @@ destination: - endpoint: somewhere:9000 # whatever your s3 endpoint is structured: true # checks repos out like hostersite/user|organization/repo bucket: your-bucket-name + use_static_creds: true # if true, use static credentials (accesskey/secretkey/token); if false, use IAM instance credentials (e.g. for AWS EC2/ECS) accesskey: your-access-key # can be an environment variable, just don't add a $ in front of it secretkey: your-secret-key # can be an environment variable, just don't add a $ in front of it token: your-token # can be an environment variable, just don't add a $ in front of it diff --git a/gickup_spec.json b/gickup_spec.json index 41e8c1c2..92a375be 100644 --- a/gickup_spec.json +++ b/gickup_spec.json @@ -531,6 +531,10 @@ "type": "string", "description": "The bucket to store the backups in" }, + "use_static_creds": { + "type": "boolean", + "description": "If true, use static credentials (accesskey/secretkey/token). If false, use IAM instance credentials (e.g. for AWS EC2/ECS). Default: false" + }, "accesskey": { "type": "string", "description": "The access key to authenticate against the S3 server" diff --git a/main.go b/main.go index fd354267..3c368dcc 100644 --- a/main.go +++ b/main.go @@ -56,7 +56,7 @@ var version = "unknown" func readConfigFile(configfile string) []*types.Conf { conf := []*types.Conf{} - cfgdata, err := os.Open(configfile) + cfgdata, err := os.Open(filepath.Clean(configfile)) if err != nil { log.Fatal(). Str("stage", "readconfig"). @@ -245,20 +245,22 @@ func backup(repos []types.Repo, conf *types.Conf) { } } - // Check if environment variables are used for accesskey and secretkey - d.AccessKey, err = d.GetKey(d.AccessKey) - if err != nil { - log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) - } - d.SecretKey, err = d.GetKey(d.SecretKey) - if err != nil { - log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) - } - if d.Token != "" { - d.Token, err = d.GetKey(d.Token) + if d.UseStaticCreds { + // Check if environment variables are used for accesskey and secretkey + d.AccessKey, err = d.GetKey(d.AccessKey) if err != nil { log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) } + d.SecretKey, err = d.GetKey(d.SecretKey) + if err != nil { + log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) + } + if d.Token != "" { + d.Token, err = d.GetKey(d.Token) + if err != nil { + log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) + } + } } if d.Zip { diff --git a/main_test.go b/main_test.go index 46486a94..da2b02f1 100644 --- a/main_test.go +++ b/main_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "path/filepath" "strings" "testing" ) @@ -40,7 +39,6 @@ func TestTildeReplacement_TildeDir(t *testing.T) { func TestReadConfigFile_InheritsPushConfigsAndExpandsHome(t *testing.T) { t.Parallel() - configPath := filepath.Join(t.TempDir(), "conf.yml") config := `destination: local: - path: "~/primary" @@ -54,10 +52,15 @@ destination: local: - path: "/tmp/secondary" ` - - if err := os.WriteFile(configPath, []byte(config), 0o644); err != nil { + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { t.Fatalf("write config: %v", err) } + f.Close() confs := readConfigFile(configPath) if len(confs) != 2 { @@ -80,3 +83,149 @@ destination: t.Fatal("expected initial push config to be populated") } } + +func TestReadConfigFile_S3UseStaticCredsTrue(t *testing.T) { + t.Parallel() + + config := `destination: + s3: + - bucket: "my-bucket" + endpoint: "s3.example.com" + use_static_creds: true + accesskey: "AKID" + secretkey: "SECRET" +` + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { + t.Fatalf("write config: %v", err) + } + f.Close() + + confs := readConfigFile(configPath) + if len(confs) != 1 { + t.Fatalf("expected 1 config, got %d", len(confs)) + } + + s3 := confs[0].Destination.S3 + if len(s3) != 1 { + t.Fatalf("expected 1 S3 destination, got %d", len(s3)) + } + + if !s3[0].UseStaticCreds { + t.Fatal("expected UseStaticCreds to be true") + } +} + +func TestReadConfigFile_S3UseStaticCredsFalse(t *testing.T) { + t.Parallel() + + config := `destination: + s3: + - bucket: "my-bucket" + endpoint: "s3.example.com" + use_static_creds: false +` + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { + t.Fatalf("write config: %v", err) + } + f.Close() + + confs := readConfigFile(configPath) + if len(confs) != 1 { + t.Fatalf("expected 1 config, got %d", len(confs)) + } + + s3 := confs[0].Destination.S3 + if len(s3) != 1 { + t.Fatalf("expected 1 S3 destination, got %d", len(s3)) + } + + if s3[0].UseStaticCreds { + t.Fatal("expected UseStaticCreds to be false") + } +} + +func TestReadConfigFile_S3UseStaticCredsResolvesEnvVars(t *testing.T) { + t.Setenv("TEST_S3_ACCESS", "resolved-access") + t.Setenv("TEST_S3_SECRET", "resolved-secret") + + config := `destination: + s3: + - bucket: "my-bucket" + endpoint: "s3.example.com" + use_static_creds: true + accesskey: "TEST_S3_ACCESS" + secretkey: "TEST_S3_SECRET" +` + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { + t.Fatalf("write config: %v", err) + } + f.Close() + + confs := readConfigFile(configPath) + s3 := confs[0].Destination.S3[0] + + // Simulate the key resolution that backup() does when UseStaticCreds is true + if !s3.UseStaticCreds { + t.Fatal("expected UseStaticCreds to be true") + } + + accessKey, err := s3.GetKey(s3.AccessKey) + if err != nil { + t.Fatalf("GetKey(accesskey) error = %v", err) + } + + secretKey, err := s3.GetKey(s3.SecretKey) + if err != nil { + t.Fatalf("GetKey(secretkey) error = %v", err) + } + + if accessKey != "resolved-access" { + t.Fatalf("accesskey = %q, want resolved-access", accessKey) + } + + if secretKey != "resolved-secret" { + t.Fatalf("secretkey = %q, want resolved-secret", secretKey) + } +} + +func TestReadConfigFile_S3UseStaticCredsAbsentSkipsKeyResolution(t *testing.T) { + t.Parallel() + + config := `destination: + s3: + - bucket: "my-bucket" + endpoint: "s3.example.com" +` + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { + t.Fatalf("write config: %v", err) + } + f.Close() + + confs := readConfigFile(configPath) + s3 := confs[0].Destination.S3[0] + + // When UseStaticCreds is false (zero value), backup() skips key resolution entirely + if s3.UseStaticCreds { + t.Fatal("expected UseStaticCreds to be false when absent from config") + } +} diff --git a/s3/s3.go b/s3/s3.go index 73689fda..b2ae9449 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -14,11 +14,19 @@ import ( var sub zerolog.Logger +// getCredentials returns the appropriate credentials based on the S3Repo configuration +func getCredentials(s3repo types.S3Repo) *credentials.Credentials { + if s3repo.UseStaticCreds { + return credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, s3repo.Token) + } + return credentials.NewIAM("") +} + // UploadDirToS3 uploads the contents of a directory to S3-compatible storage func UploadDirToS3(directory string, s3repo types.S3Repo, options *minio.PutObjectOptions) error { // Initialize minio client object. client, err := minio.New(s3repo.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, s3repo.Token), + Creds: getCredentials(s3repo), Secure: s3repo.UseSSL, Region: s3repo.Region, }) @@ -71,7 +79,7 @@ func DeleteObjectsNotInRepo(directory, bucketdir string, s3repo types.S3Repo) er sub = logger.CreateSubLogger("stage", "s3", "endpoint", s3repo.Endpoint, "bucket", s3repo.Bucket) // Initialize minio client object. client, err := minio.New(s3repo.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, s3repo.Token), + Creds: getCredentials(s3repo), Secure: s3repo.UseSSL, Region: s3repo.Region, }) diff --git a/s3/s3_test.go b/s3/s3_test.go new file mode 100644 index 00000000..d52b5a4f --- /dev/null +++ b/s3/s3_test.go @@ -0,0 +1,49 @@ +package s3 + +import ( + "testing" + + "github.com/cooperspencer/gickup/types" +) + +func TestGetCredentialsStaticCredsReturnsStaticValues(t *testing.T) { + t.Parallel() + + s3repo := types.S3Repo{ + UseStaticCreds: true, + AccessKey: "test-access", + SecretKey: "test-secret", + Token: "test-token", + } + + creds := getCredentials(s3repo) + if creds == nil { + t.Fatal("expected non-nil credentials") + } + + val, err := creds.Get() + if err != nil { + t.Fatalf("Get() error = %v", err) + } + + if val.AccessKeyID != "test-access" { + t.Fatalf("AccessKeyID = %q, want %q", val.AccessKeyID, "test-access") + } + + if val.SecretAccessKey != "test-secret" { + t.Fatalf("SecretAccessKey = %q, want %q", val.SecretAccessKey, "test-secret") + } + + if val.SessionToken != "test-token" { + t.Fatalf("SessionToken = %q, want %q", val.SessionToken, "test-token") + } +} + +func TestGetCredentialsIAMReturnsNonNil(t *testing.T) { + t.Parallel() + + creds := getCredentials(types.S3Repo{UseStaticCreds: false}) + if creds == nil { + t.Fatal("expected non-nil IAM credentials") + } +} diff --git a/types/types.go b/types/types.go index 31ad9c91..a4f126c5 100644 --- a/types/types.go +++ b/types/types.go @@ -552,6 +552,7 @@ func StatRemote(remoteURL, sshURL string, repo GenRepo) bool { type S3Repo struct { Bucket string `yaml:"bucket"` Endpoint string `yaml:"endpoint"` + UseStaticCreds bool `yaml:"use_static_creds" default:"true"` AccessKey string `yaml:"accesskey"` SecretKey string `yaml:"secretkey"` Token string `yaml:"token"` diff --git a/types/types_test.go b/types/types_test.go index d70f7a0d..9060a6a9 100644 --- a/types/types_test.go +++ b/types/types_test.go @@ -308,3 +308,21 @@ func TestResolveTrimsNothingFromLiteral(t *testing.T) { t.Fatalf("resolve() = %q, want already-set", got) } } + +func TestS3RepoUseStaticCredsDefaultFalse(t *testing.T) { + t.Parallel() + + var s3 S3Repo + if s3.UseStaticCreds { + t.Fatal("expected UseStaticCreds to be false by default (Go zero value)") + } +} + +func TestS3RepoUseStaticCredsExplicitTrue(t *testing.T) { + t.Parallel() + + s3 := S3Repo{UseStaticCreds: true} + if !s3.UseStaticCreds { + t.Fatal("expected UseStaticCreds to be true when explicitly set") + } +} From c8a812e5c3c301cb9c16823fbc7ddc2adfad23bf Mon Sep 17 00:00:00 2001 From: Hildebrand Soethoudt <54576297+Hildebrand-Ritense@users.noreply.github.com> Date: Fri, 12 Jun 2026 08:23:29 +0200 Subject: [PATCH 12/44] Rework of BitBucket authentication (#402) * Rework of the BitBucket source authentication: * Renamed 'user' to 'organization' as it better reflects the bitbucket architecture * Added email config field, used fetch the workspace repo list with API token auth * Added error log line in case workspace repo list fetch fails * go-bitbucket version bump to 0.10.0 * fixed linting error * ran go mod tidy to cleanup unused deps --- bitbucket/bitbucket.go | 17 +++++++++++++---- go.mod | 10 +++++----- go.sum | 24 ++++++++++++------------ types/types.go | 1 + 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/bitbucket/bitbucket.go b/bitbucket/bitbucket.go index 796f6d71..47d7811c 100644 --- a/bitbucket/bitbucket.go +++ b/bitbucket/bitbucket.go @@ -2,6 +2,7 @@ package bitbucket import ( "net/url" + "os" "time" "github.com/cooperspencer/gickup/logger" @@ -35,7 +36,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { includeorgs := types.GetMap(repo.IncludeOrgs) excludeorgs := types.GetMap(repo.ExcludeOrgs) - client, err := bitbucket.NewBasicAuth(repo.Username, repo.Password) + client, err := bitbucket.NewBasicAuth(repo.Email, repo.Password) if repo.URL == "" { repo.URL = bitbucket.DEFAULT_BITBUCKET_API_BASE_URL @@ -64,7 +65,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { sub.Info(). Msgf("grabbing repositories from %s", repo.User) - repoRes, err := client.Repositories.ListForAccount(&bitbucket.RepositoriesOptions{Owner: repo.User}) + repoRes, err := client.Repositories.ListForAccount(&bitbucket.RepositoriesOptions{Owner: repo.Organization}) var repositories []bitbucket.Repository if err == nil { repositories = repoRes.Items @@ -95,6 +96,9 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } } } + } else { + sub.Error().Msgf("Exiting due to error fetching Bitbucket workspace repositories: %s", err.Error()) + os.Exit(1) } for _, r := range repositories { @@ -129,13 +133,18 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } + origin := repo + if repo.Username != "" { + origin.User = repo.Username + } + if include[r.Name] { repos = append(repos, types.Repo{ Name: r.Name, URL: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string), SSHURL: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string), Token: repo.Token, - Origin: repo, + Origin: origin, Owner: user, Hoster: types.GetHost(repo.URL), Description: r.Description, @@ -151,7 +160,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { URL: r.Links["clone"].([]interface{})[0].(map[string]interface{})["href"].(string), SSHURL: r.Links["clone"].([]interface{})[1].(map[string]interface{})["href"].(string), Token: repo.Token, - Origin: repo, + Origin: origin, Owner: user, Hoster: types.GetHost(repo.URL), Description: r.Description, diff --git a/go.mod b/go.mod index dc47db53..dd2c4de0 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/google/go-github/v74 v74.0.0 github.com/gookit/color v1.6.0 github.com/hasura/go-graphql-client v0.16.0 - github.com/ktrysmt/go-bitbucket v0.9.100 + github.com/ktrysmt/go-bitbucket v0.10.0 github.com/melbahja/goph v1.5.0 github.com/minio/minio-go/v7 v7.0.100 github.com/prometheus/client_golang v1.23.2 @@ -24,7 +24,7 @@ require ( github.com/rs/zerolog v1.35.0 github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed gitlab.com/gitlab-org/api/client-go v1.46.0 - golang.org/x/crypto v0.50.0 + golang.org/x/crypto v0.51.0 golang.org/x/oauth2 v0.36.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -85,9 +85,9 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.53.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 74b07b81..edbd22bb 100644 --- a/go.sum +++ b/go.sum @@ -131,8 +131,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/ktrysmt/go-bitbucket v0.9.100 h1:d0CWilBnJ2ywomeH0/1QNULsEUziye/AD0cOEKfCMx4= -github.com/ktrysmt/go-bitbucket v0.9.100/go.mod h1:/1H7KMC7FsdkRoNMq21an6dyFpA+irNw5tyYpHzRC2Q= +github.com/ktrysmt/go-bitbucket v0.10.0 h1:miXSfqFb+j0pPBB9h8kw9f/oeq045WOPI4hplr996+k= +github.com/ktrysmt/go-bitbucket v0.10.0/go.mod h1:IUB8I+gC3UO00NjNTMS7STjsZYq+EAhPuHgynzRWxqY= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -213,15 +213,15 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= -golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -233,16 +233,16 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= -golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/types/types.go b/types/types.go index a4f126c5..12182622 100644 --- a/types/types.go +++ b/types/types.go @@ -249,6 +249,7 @@ type GenRepo struct { Token string `yaml:"token"` TokenFile string `yaml:"token_file"` User string `yaml:"user"` + Email string `yaml:"email"` Organization string `yaml:"organization"` SSH bool `yaml:"ssh"` SSHKey string `yaml:"sshkey"` From d5caefb31066d553dc4c00efb3e723b410000197 Mon Sep 17 00:00:00 2001 From: Bernat Mut <1116133+monwolf@users.noreply.github.com> Date: Tue, 16 Jun 2026 14:12:59 +0200 Subject: [PATCH 13/44] Feature/discussion 391 add GitHub app support (#396) * feat: Add GitHub App support for authentication in configuration and code * feat: Enhance GitHub App support by updating repository fetching logic and authentication method * test: Update dependencies, enhance configuration handling, and add GitHub client tests - Updated dependencies in go.sum to latest versions for improved stability and security. - Added a defer statement to close the configuration file in readConfigFile function to prevent resource leaks. - Implemented unit tests for GenRepo's HasAppAuth method to validate authentication conditions. - Created new tests for the GitHub client to verify behavior with and without authentication tokens, including handling of invalid and missing private key files. * feat: Exclude G101 gosec warning for github/github_test.go * fixed go.mod and go.sum --------- Co-authored-by: Andreas Wachter --- .golangci.yml | 3 ++ conf.example.yml | 15 +++++++ gickup_spec.json | 48 +++++++++++++++++++++ github/github.go | 98 ++++++++++++++++++++++++++++++------------- github/github_test.go | 84 +++++++++++++++++++++++++++++++++++++ go.mod | 24 ++++++----- go.sum | 63 +++++++++++++++++++--------- local/local.go | 39 +++++++++-------- local/local_test.go | 52 +++++++++++++++++++++++ main.go | 1 + types/types.go | 60 ++++++++++++++------------ types/types_test.go | 64 ++++++++++++++++++++++++++++ 12 files changed, 445 insertions(+), 106 deletions(-) create mode 100644 github/github_test.go diff --git a/.golangci.yml b/.golangci.yml index 5ea7704e..2f307f1f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -64,6 +64,9 @@ linters: text: "G117:" - linters: [gosec] text: "G703:" + - linters: [gosec] + path: github/github_test.go + text: "G101:" # Global exclusions for staticcheck style suggestions - linters: [staticcheck] text: "SA1019:" diff --git a/conf.example.yml b/conf.example.yml index 1a159b45..4369a9f3 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -38,6 +38,13 @@ source: - java excludeforks: true # exclude forked repositories gists: true # clone gists too + # alternatively, authenticate with a GitHub App: + # - app_id: 123456 # GitHub App ID (numeric) + # app_installation_id: 78901234 # Installation ID of the App on the target account + # app_private_key_file: /path/to/private.pem # path to the App's RSA private key PEM file + # user: some-user # user or org whose repos to back up (optional if App is self-installed) + # organization: some-org # alternatively back up an organization's repos + # ssh: false # note: 'contributed' is not supported with App auth gitea: - token: some-token # token_file: token.txt # alternatively, specify token in a file @@ -247,6 +254,14 @@ destination: visibility: repositories: private # private, public, default: private force: false # force push to destination + # alternatively, authenticate with a GitHub App: + # - app_id: 123456 # GitHub App ID (numeric) + # app_installation_id: 78901234 # Installation ID of the App + # app_private_key_file: /path/to/private.pem # path to the App's RSA private key PEM file + # organization: whatever # org to back up into (leave empty for personal account) + # visibility: + # repositories: private + # force: false onedev: - token: some-token # token_file: token.txt # alternatively, specify token in a file diff --git a/gickup_spec.json b/gickup_spec.json index 92a375be..9a50d8f9 100644 --- a/gickup_spec.json +++ b/gickup_spec.json @@ -70,6 +70,15 @@ "gists": { "$ref": "#/definitions/source/properties/gists" }, + "app_id": { + "$ref": "#/definitions/source/properties/app_id" + }, + "app_installation_id": { + "$ref": "#/definitions/source/properties/app_installation_id" + }, + "app_private_key_file": { + "$ref": "#/definitions/source/properties/app_private_key_file" + }, "filter": { "$ref": "#/definitions/filter" } @@ -588,6 +597,15 @@ "force": { "$ref": "#/definitions/destination/properties/force" }, + "app_id": { + "$ref": "#/definitions/destination/properties/app_id" + }, + "app_installation_id": { + "$ref": "#/definitions/destination/properties/app_installation_id" + }, + "app_private_key_file": { + "$ref": "#/definitions/destination/properties/app_private_key_file" + }, "visibility": { "type": "object", "properties": { @@ -880,6 +898,21 @@ "$id": "#/definitions/source/properties/gists", "type": "boolean", "description": "Include the gists in the backup" + }, + "app_id": { + "$id": "#/definitions/source/properties/app_id", + "type": "integer", + "description": "GitHub App ID (numeric). Used together with app_installation_id and app_private_key_file to authenticate as a GitHub App." + }, + "app_installation_id": { + "$id": "#/definitions/source/properties/app_installation_id", + "type": "integer", + "description": "Installation ID of the GitHub App on the target account or organization." + }, + "app_private_key_file": { + "$id": "#/definitions/source/properties/app_private_key_file", + "type": "string", + "description": "Path to the RSA private key PEM file for the GitHub App." } }, "additionalProperties": false @@ -932,6 +965,21 @@ "$id": "#/definitions/destination/properties/force", "type": "boolean", "description": "Force push the repositories" + }, + "app_id": { + "$id": "#/definitions/destination/properties/app_id", + "type": "integer", + "description": "GitHub App ID (numeric). Used together with app_installation_id and app_private_key_file to authenticate as a GitHub App." + }, + "app_installation_id": { + "$id": "#/definitions/destination/properties/app_installation_id", + "type": "integer", + "description": "Installation ID of the GitHub App on the target account or organization." + }, + "app_private_key_file": { + "$id": "#/definitions/destination/properties/app_private_key_file", + "type": "string", + "description": "Path to the RSA private key PEM file for the GitHub App." } }, "additionalProperties": false diff --git a/github/github.go b/github/github.go index 581e1e49..cad93685 100644 --- a/github/github.go +++ b/github/github.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/bradleyfalzon/ghinstallation/v2" "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" "github.com/google/go-github/v74/github" @@ -47,6 +48,34 @@ type V4Repo struct { var sub zerolog.Logger +// newGithubClient creates an authenticated GitHub client. +// When App authentication fields are configured it uses a GitHub App installation token; +// otherwise it falls back to a personal access token or an unauthenticated client. +// It returns the client, the access token string (for git operations), and any error. +func newGithubClient(ctx context.Context, repo types.GenRepo) (*github.Client, string, error) { + if repo.HasAppAuth() { + itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, repo.AppID, repo.AppInstallationID, repo.AppPrivateKeyFile) + if err != nil { + return nil, "", fmt.Errorf("github app auth: %w", err) + } + client := github.NewClient(&http.Client{Transport: itr}) + token, err := itr.Token(ctx) + if err != nil { + return nil, "", fmt.Errorf("github app token: %w", err) + } + return client, token, nil + } + + token := repo.GetToken() + if token == "" { + return github.NewClient(nil), "", nil + } + + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) + tc := oauth2.NewClient(ctx, ts) + return github.NewClient(tc), token, nil +} + func getv4(token, user string) []V4Repo { repos := []V4Repo{} tokenSource := oauth2.StaticTokenSource( @@ -128,22 +157,15 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { i := 1 githubrepos := []*github.Repository{} - token := repo.GetToken() - - var client *github.Client - if token == "" { - client = github.NewClient(nil) - } else { - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(context.TODO(), ts) - client = github.NewClient(tc) + client, token, err := newGithubClient(context.TODO(), repo) + if err != nil { + sub.Error().Msg(err.Error()) + continue } v4user := repo.User - if token != "" { + if token != "" && !repo.HasAppAuth() { user, _, err := client.Users.Get(context.TODO(), "") if err != nil { sub.Error(). @@ -158,20 +180,38 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } if token != "" && v4user != "" && repo.Contributed { - for _, r := range getv4(token, v4user) { - githubRepo, _, err := client.Repositories.Get(context.Background(), r.User, r.Repository) - if err != nil { - sub.Error(). - Msg(err.Error()) - continue + if repo.HasAppAuth() { + sub.Warn().Msg("contributed repos are not supported with GitHub App authentication, skipping") + } else { + for _, r := range getv4(token, v4user) { + githubRepo, _, err := client.Repositories.Get(context.Background(), r.User, r.Repository) + if err != nil { + sub.Error(). + Msg(err.Error()) + continue + } + githubrepos = append(githubrepos, githubRepo) } - githubrepos = append(githubrepos, githubRepo) } } for { opt.Page = i - repos, status, err := client.Repositories.List(context.TODO(), repo.User, opt) + var fetchedRepos []*github.Repository + var status *github.Response + var err error + + if repo.HasAppAuth() { + appListOpt := &github.ListOptions{Page: i, PerPage: opt.PerPage} + var result *github.ListRepositories + result, status, err = client.Apps.ListRepos(context.TODO(), appListOpt) + if result != nil { + fetchedRepos = result.Repositories + } + } else { + fetchedRepos, status, err = client.Repositories.List(context.TODO(), repo.User, opt) + } + if err != nil { sub.Error(). Msg(err.Error()) @@ -188,14 +228,15 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { continue } - if status.StatusCode == http.StatusNotFound { + if status != nil && status.StatusCode == http.StatusNotFound { break } + break } - if len(repos) == 0 { + if len(fetchedRepos) == 0 { break } - githubrepos = append(githubrepos, repos...) + githubrepos = append(githubrepos, fetchedRepos...) i++ } @@ -378,13 +419,10 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { // GetOrCreate Get or create a repository func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com") - token := destination.GetToken() - ts := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: token}, - ) - tc := oauth2.NewClient(context.TODO(), ts) - - client := github.NewClient(tc) + client, _, err := newGithubClient(context.TODO(), destination) + if err != nil { + return "", err + } dest := types.GithubDestination{} login := "" diff --git a/github/github_test.go b/github/github_test.go new file mode 100644 index 00000000..46ebc835 --- /dev/null +++ b/github/github_test.go @@ -0,0 +1,84 @@ +package github + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/cooperspencer/gickup/types" +) + +func TestNewGithubClientUnauthenticated(t *testing.T) { + t.Parallel() + + repo := types.GenRepo{} // no token, no app auth + + client, token, err := newGithubClient(context.Background(), repo) + if err != nil { + t.Fatalf("newGithubClient() error = %v", err) + } + + if client == nil { + t.Fatal("expected non-nil client") + } + + if token != "" { + t.Fatalf("expected empty token, got %q", token) + } +} + +func TestNewGithubClientWithToken(t *testing.T) { + t.Setenv("GITHUB_TEST_TOKEN", "my-personal-token") + + repo := types.GenRepo{Token: "GITHUB_TEST_TOKEN"} + + client, token, err := newGithubClient(context.Background(), repo) + if err != nil { + t.Fatalf("newGithubClient() error = %v", err) + } + + if client == nil { + t.Fatal("expected non-nil client") + } + + if token != "my-personal-token" { + t.Fatalf("token = %q, want %q", token, "my-personal-token") + } +} + +func TestNewGithubClientAppAuthInvalidKeyFile(t *testing.T) { + t.Parallel() + + // A file that exists but contains an invalid PEM key. + keyFile := filepath.Join(t.TempDir(), "invalid.pem") + if err := os.WriteFile(keyFile, []byte("not-a-valid-pem-key"), 0o600); err != nil { + t.Fatalf("write key file: %v", err) + } + + repo := types.GenRepo{ + AppID: 1, + AppInstallationID: 2, + AppPrivateKeyFile: keyFile, + } + + _, _, err := newGithubClient(context.Background(), repo) + if err == nil { + t.Fatal("expected error for invalid App private key file") + } +} + +func TestNewGithubClientAppAuthMissingKeyFile(t *testing.T) { + t.Parallel() + + repo := types.GenRepo{ + AppID: 1, + AppInstallationID: 2, + AppPrivateKeyFile: "/nonexistent/path/key.pem", + } + + _, _, err := newGithubClient(context.Background(), repo) + if err == nil { + t.Fatal("expected error when App private key file does not exist") + } +} diff --git a/go.mod b/go.mod index dd2c4de0..1a14a5da 100644 --- a/go.mod +++ b/go.mod @@ -3,25 +3,26 @@ module github.com/cooperspencer/gickup go 1.26 require ( - code.gitea.io/sdk/gitea v0.24.1 + code.gitea.io/sdk/gitea v0.25.1 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.1 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.1 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 github.com/alecthomas/kong v1.15.0 + github.com/bradleyfalzon/ghinstallation/v2 v2.18.0 github.com/cooperspencer/onedev v0.0.0-20240801121041-919f6d3f7ea6 github.com/go-git/go-git/v5 v5.19.1 github.com/goccy/go-yaml v1.19.2 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/google/go-cmp v0.7.0 github.com/google/go-github/v74 v74.0.0 - github.com/gookit/color v1.6.0 + github.com/gookit/color v1.6.1 github.com/hasura/go-graphql-client v0.16.0 github.com/ktrysmt/go-bitbucket v0.10.0 github.com/melbahja/goph v1.5.0 github.com/minio/minio-go/v7 v7.0.100 github.com/prometheus/client_golang v1.23.2 github.com/robfig/cron/v3 v3.0.1 - github.com/rs/zerolog v1.35.0 + github.com/rs/zerolog v1.35.1 github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed gitlab.com/gitlab-org/api/client-go v1.46.0 golang.org/x/crypto v0.51.0 @@ -33,7 +34,7 @@ require ( dario.cat/mergo v1.0.2 // indirect github.com/42wim/httpsig v1.2.4 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/ProtonMail/go-crypto v1.4.1 // indirect github.com/beorn7/perks v1.0.1 // indirect @@ -48,8 +49,10 @@ require ( github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.9.0 // indirect github.com/go-ini/ini v1.67.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/go-github/v84 v84.0.0 // indirect github.com/google/go-querystring v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -57,13 +60,13 @@ require ( github.com/hashicorp/go-version v1.9.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.6.0 // indirect - github.com/klauspost/compress v1.18.5 // indirect + github.com/klauspost/compress v1.18.6 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/klauspost/crc32 v1.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.21 // indirect + github.com/mattn/go-colorable v0.1.15 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect github.com/minio/crc64nvme v1.1.1 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -74,7 +77,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pkg/sftp v1.13.10 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/common v0.68.0 // indirect github.com/prometheus/procfs v0.20.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sergi/go-diff v1.4.0 // indirect @@ -83,12 +86,13 @@ require ( github.com/tinylib/msgp v1.6.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - go.yaml.in/yaml/v2 v2.4.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/ini.v1 v1.67.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index edbd22bb..fa207204 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -code.gitea.io/sdk/gitea v0.24.1 h1:hpaqcdGcBmfMpV7JSbBJVwE99qo+WqGreJYKrDKEyW8= -code.gitea.io/sdk/gitea v0.24.1/go.mod h1:5/77BL3sHneCMEiZaMT9lfTvnnibsYxyO48mceCF3qA= +code.gitea.io/sdk/gitea v0.25.1 h1:yywxWwoV+SdjHtbC6unBiXojWdZOtoHuGhEazEXeWuE= +code.gitea.io/sdk/gitea v0.25.1/go.mod h1:uDFWYBU8dgZsgOHwe6C/6olxvf8FHguNB3wW1i83fgg= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/42wim/httpsig v1.2.4 h1:mI5bH0nm4xn7K18fo1K3okNDRq8CCJ0KbBYWyA6r8lU= @@ -14,12 +14,12 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6Xu github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0 h1:BM85pSYlVYQHdq00nxyPoOkyLF5NArJG3bOsrmbwr4k= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.7.0/go.mod h1:QYjP2cB7ZYtS/8jAbE0VSBZde/tjExqGjp+8JY6/+ts= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1 h1:edShSHV3DV90+kt+CMaEXEzR9QF7wFrPJxVGz2blMIU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.7.1/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2 h1:RHK7bS+HQMslb1sZpAokUt+zTVmue0hKSs2C791hhzU= +github.com/AzureAD/microsoft-authentication-library-for-go v1.7.2/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -37,6 +37,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bradleyfalzon/ghinstallation/v2 v2.18.0 h1:WPqnN6NS9XvYlOgZQAIseN7Z1uAiE+UxgDKlW7FvFuU= +github.com/bradleyfalzon/ghinstallation/v2 v2.18.0/go.mod h1:gpoSwwWc4biE49F7n+roCcpkEkZ1Qr9soZ2ESvMiouU= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/circl v1.6.3 h1:9GPOhQGF9MCYUeXyMYlqTR6a5gTrgR/fBLXvUgtVcg8= @@ -58,8 +60,8 @@ github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= @@ -78,6 +80,8 @@ github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 h1:UjoPNDAQ5JPCjlxoJd6K8ALZqSDDhk2ymieAZOVaDg0= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85/go.mod h1:fR6z1Ie6rtF7kl/vBYMfgD5/G5B1blui7z426/sj2DU= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= @@ -87,14 +91,16 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v74 v74.0.0 h1:yZcddTUn8DPbj11GxnMrNiAnXH14gNs559AsUpNpPgM= github.com/google/go-github/v74 v74.0.0/go.mod h1:ubn/YdyftV80VPSI26nSJvaEsTOnsjrxG3o9kJhcyak= +github.com/google/go-github/v84 v84.0.0 h1:I/0Xn5IuChMe8TdmI2bbim5nyhaRFJ7DEdzmD2w+yVA= +github.com/google/go-github/v84 v84.0.0/go.mod h1:WwYL1z1ajRdlaPszjVu/47x1L0PXukJBn73xsiYrRRQ= github.com/google/go-querystring v1.2.0 h1:yhqkPbu2/OH+V9BfpCVPZkNmUXhb2gBxJArfhIxNtP0= github.com/google/go-querystring v1.2.0/go.mod h1:8IFJqpSRITyJ8QhQ13bmbeMBDfmeEJZD5A0egEOmkqU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gookit/assert v0.1.1 h1:lh3GcawXe/p+cU7ESTZ5Ui3Sm/x8JWpIis4/1aF0mY0= github.com/gookit/assert v0.1.1/go.mod h1:jS5bmIVQZTIwk42uXl4lyj4iaaxx32tqH16CFj0VX2E= -github.com/gookit/color v1.6.0 h1:JjJXBTk1ETNyqyilJhkTXJYYigHG24TM9Xa2M1xAhRA= -github.com/gookit/color v1.6.0/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= +github.com/gookit/color v1.6.1 h1:KoTnDxJPRgrL0SoX0f8rCFg2zI0t4E3GZZBMo2nN8LU= +github.com/gookit/color v1.6.1/go.mod h1:9ACFc7/1IpHGBW8RwuDm/0YEnhg3dwwXpoMsmtyHfjs= github.com/graph-gophers/graphql-go v1.9.0 h1:yu0ucKHLc5qGpRwLYKIWtr9bOoxovkWasuBrPQwlHls= github.com/graph-gophers/graphql-go v1.9.0/go.mod h1:23olKZ7duEvHlF/2ELEoSZaY1aNPfShjP782SOoNTyM= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -115,8 +121,8 @@ github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRg github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= -github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= -github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -135,18 +141,22 @@ github.com/ktrysmt/go-bitbucket v0.10.0 h1:miXSfqFb+j0pPBB9h8kw9f/oeq045WOPI4hpl github.com/ktrysmt/go-bitbucket v0.10.0/go.mod h1:IUB8I+gC3UO00NjNTMS7STjsZYq+EAhPuHgynzRWxqY= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= -github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= -github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= -github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw4= +github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/melbahja/goph v1.5.0 h1:RQUBpLvfg3i7fjfG8rTcSWyMjVRfdhwrrfQhjYee4dQ= github.com/melbahja/goph v1.5.0/go.mod h1:dDwo+44cmvfDLdiVpc6fJxexf5BA5yEDUeE5YgtuDO4= +github.com/melbahja/goph v1.5.1 h1:gdZyjn5nM6FO6mAZrv7XqE2gArQbb+V6FdIyH3Ru3vA= +github.com/melbahja/goph v1.5.1/go.mod h1:dDwo+44cmvfDLdiVpc6fJxexf5BA5yEDUeE5YgtuDO4= github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.100 h1:ShkWi8Tyj9RtU57OQB2HIXKz4bFgtVib0bbT1sbtLI8= github.com/minio/minio-go/v7 v7.0.100/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= +github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs= +github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -169,8 +179,8 @@ github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= -github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/common v0.68.0 h1:8rQJvQmYltsR2L7h8Zw0Iyj8WYNNmpwikoQTZXwfVeA= +github.com/prometheus/common v0.68.0/go.mod h1:4soH+U8yJSROk7OJ//hmTiWKsxapv6zRGgTt3keN8gQ= github.com/prometheus/procfs v0.20.1 h1:XwbrGOIplXW/AU3YhIhLODXMJYyC1isLFfYCsTEycfc= github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4UlK79eF5TGGo= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -179,8 +189,8 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= -github.com/rs/zerolog v1.35.0 h1:VD0ykx7HMiMJytqINBsKcbLS+BJ4WYjz+05us+LRTdI= -github.com/rs/zerolog v1.35.0/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= +github.com/rs/zerolog v1.35.1 h1:m7xQeoiLIiV0BCEY4Hs+j2NG4Gp2o2KPKmhnnLiazKI= +github.com/rs/zerolog v1.35.1/go.mod h1:EjML9kdfa/RMA7h/6z6pYmq1ykOuA8/mjWaEvGI+jcw= github.com/sergi/go-diff v1.4.0 h1:n/SP9D5ad1fORl+llWyN+D6qoUETXNZARKjyY2/KVCw= github.com/sergi/go-diff v1.4.0/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed h1:KT7hI8vYXgU0s2qaMkrfq9tCA1w/iEPgfredVP+4Tzw= @@ -191,8 +201,14 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg= github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= @@ -201,6 +217,10 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= gitlab.com/gitlab-org/api/client-go v1.46.0 h1:YxBWFZIFYKcGESCb9fpkwzouo+apyB9pr/XTWzNoL24= gitlab.com/gitlab-org/api/client-go v1.46.0/go.mod h1:FtgyU6g2HS5+fMhw6nLK96GBEEBx5MzntOiJWfIaiN8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -252,11 +272,14 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= +gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/local/local.go b/local/local.go index 94d3f7a3..ca1e5819 100644 --- a/local/local.go +++ b/local/local.go @@ -34,6 +34,23 @@ var ( sub zerolog.Logger ) +// tokenAuth returns the HTTP basic auth for a repo that authenticates via token. +func tokenAuth(repo types.Repo) *http.BasicAuth { + if repo.Token == "" { + return nil + } + if repo.NoTokenUser { + return &http.BasicAuth{ + Username: "x-access-token", + Password: repo.Token, + } + } + return &http.BasicAuth{ + Username: repo.Origin.User, + Password: repo.Token, + } +} + // Locally TODO. func Locally(repo types.Repo, l types.Local, dry bool) bool { sub = logger.CreateSubLogger("stage", "locally", "path", l.Path) @@ -97,16 +114,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { return false } case repo.Token != "": - if repo.NoTokenUser { - auth = &http.BasicAuth{ - Username: repo.Token, - } - } else { - auth = &http.BasicAuth{ - Username: repo.Origin.User, - Password: repo.Token, - } - } + auth = tokenAuth(repo) case repo.Origin.Username != "" && repo.Origin.Password != "": auth = &http.BasicAuth{ Username: repo.Origin.Username, @@ -519,16 +527,7 @@ func TempCloneBare(repo types.Repo, tempdir string) (*git.Repository, error) { func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repository, error) { var auth transport.AuthMethod if repo.Token != "" { - if repo.NoTokenUser { - auth = &http.BasicAuth{ - Username: repo.Token, - } - } else { - auth = &http.BasicAuth{ - Username: repo.Origin.User, - Password: repo.Token, - } - } + auth = tokenAuth(repo) } if repo.Origin.LFS { g, err := gitcmd.New() diff --git a/local/local_test.go b/local/local_test.go index f1a1be96..fa1d730f 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -3,6 +3,8 @@ package local import ( "strings" "testing" + + "github.com/cooperspencer/gickup/types" ) func TestRandomStringLengthAndCharset(t *testing.T) { @@ -29,3 +31,53 @@ func TestRandomStringZeroLength(t *testing.T) { t.Fatalf("RandomString(0) = %q, want empty string", got) } } + +func TestTokenAuth_NoToken(t *testing.T) { + t.Parallel() + + repo := types.Repo{Token: ""} + if got := tokenAuth(repo); got != nil { + t.Fatalf("tokenAuth with empty token = %v, want nil", got) + } +} + +func TestTokenAuth_NoTokenUser(t *testing.T) { + t.Parallel() + + repo := types.Repo{ + Token: "mytoken", + NoTokenUser: true, + } + got := tokenAuth(repo) + if got == nil { + t.Fatal("tokenAuth returned nil, want *BasicAuth") + } + if got.Username != "x-access-token" { + t.Errorf("Username = %q, want %q", got.Username, "x-access-token") + } + if got.Password != "mytoken" { + t.Errorf("Password = %q, want %q", got.Password, "mytoken") + } +} + +func TestTokenAuth_WithTokenUser(t *testing.T) { + t.Parallel() + + repo := types.Repo{ + Token: "mytoken", + NoTokenUser: false, + Origin: types.GenRepo{ + User: "octocat", + }, + } + got := tokenAuth(repo) + if got == nil { + t.Fatal("tokenAuth returned nil, want *BasicAuth") + } + if got.Username != "octocat" { + t.Errorf("Username = %q, want %q", got.Username, "octocat") + } + if got.Password != "mytoken" { + t.Errorf("Password = %q, want %q", got.Password, "mytoken") + } +} diff --git a/main.go b/main.go index 3c368dcc..2c828f2a 100644 --- a/main.go +++ b/main.go @@ -63,6 +63,7 @@ func readConfigFile(configfile string) []*types.Conf { Str("file", configfile). Msgf("Cannot open config file from %s", types.Red(configfile)) } + defer cfgdata.Close() dec := yaml.NewDecoder(cfgdata) diff --git a/types/types.go b/types/types.go index 12182622..01a12f0b 100644 --- a/types/types.go +++ b/types/types.go @@ -246,32 +246,35 @@ func (source Source) Count() int { // GenRepo Generell Repo. type GenRepo struct { - Token string `yaml:"token"` - TokenFile string `yaml:"token_file"` - User string `yaml:"user"` - Email string `yaml:"email"` - Organization string `yaml:"organization"` - SSH bool `yaml:"ssh"` - SSHKey string `yaml:"sshkey"` - Username string `yaml:"username"` - Password string `yaml:"password"` - URL string `yaml:"url"` - Exclude []string `yaml:"exclude"` - ExcludeOrgs []string `yaml:"excludeorgs"` - Include []string `yaml:"include"` - IncludeOrgs []string `yaml:"includeorgs"` - Issues bool `yaml:"issues"` - Wiki bool `yaml:"wiki"` - Starred bool `yaml:"starred"` - CreateOrg bool `yaml:"createorg"` - Visibility Visibility `yaml:"visibility"` - Filter Filter `yaml:"filter"` - Force bool `yaml:"force"` - Contributed bool `yaml:"contributed"` - MirrorInterval string `yaml:"mirrorinterval"` - LFS bool `yaml:"lfs"` - Mirror Mirror `yaml:"mirror"` - Gists bool `yaml:"gists"` + Token string `yaml:"token"` + TokenFile string `yaml:"token_file"` + User string `yaml:"user"` + Email string `yaml:"email"` + Organization string `yaml:"organization"` + SSH bool `yaml:"ssh"` + SSHKey string `yaml:"sshkey"` + Username string `yaml:"username"` + Password string `yaml:"password"` + URL string `yaml:"url"` + Exclude []string `yaml:"exclude"` + ExcludeOrgs []string `yaml:"excludeorgs"` + Include []string `yaml:"include"` + IncludeOrgs []string `yaml:"includeorgs"` + Issues bool `yaml:"issues"` + Wiki bool `yaml:"wiki"` + Starred bool `yaml:"starred"` + CreateOrg bool `yaml:"createorg"` + Visibility Visibility `yaml:"visibility"` + Filter Filter `yaml:"filter"` + Force bool `yaml:"force"` + Contributed bool `yaml:"contributed"` + MirrorInterval string `yaml:"mirrorinterval"` + LFS bool `yaml:"lfs"` + Mirror Mirror `yaml:"mirror"` + Gists bool `yaml:"gists"` + AppID int64 `yaml:"app_id"` + AppInstallationID int64 `yaml:"app_installation_id"` + AppPrivateKeyFile string `yaml:"app_private_key_file"` } // Mirror struct @@ -309,6 +312,11 @@ func (grepo GenRepo) GetToken() string { return token } +// HasAppAuth returns true when all GitHub App authentication fields are set. +func (grepo GenRepo) HasAppAuth() bool { + return grepo.AppID > 0 && grepo.AppInstallationID > 0 && grepo.AppPrivateKeyFile != "" +} + func (f *Filter) ParseDuration() error { rest := strings.Trim(f.LastActivityString, " ") date := time.Now() diff --git a/types/types_test.go b/types/types_test.go index 9060a6a9..44008392 100644 --- a/types/types_test.go +++ b/types/types_test.go @@ -249,6 +249,70 @@ func TestSiteGetValuesSSHURL(t *testing.T) { } } +func TestHasAppAuthReturnsTrueWhenAllFieldsSet(t *testing.T) { + t.Parallel() + + repo := GenRepo{ + AppID: 12345, + AppInstallationID: 67890, + AppPrivateKeyFile: "/path/to/key.pem", + } + + if !repo.HasAppAuth() { + t.Fatal("expected HasAppAuth() = true when all fields are set") + } +} + +func TestHasAppAuthReturnsFalseWhenAppIDMissing(t *testing.T) { + t.Parallel() + + repo := GenRepo{ + AppID: 0, + AppInstallationID: 67890, + AppPrivateKeyFile: "/path/to/key.pem", + } + + if repo.HasAppAuth() { + t.Fatal("expected HasAppAuth() = false when AppID is zero") + } +} + +func TestHasAppAuthReturnsFalseWhenInstallationIDMissing(t *testing.T) { + t.Parallel() + + repo := GenRepo{ + AppID: 12345, + AppInstallationID: 0, + AppPrivateKeyFile: "/path/to/key.pem", + } + + if repo.HasAppAuth() { + t.Fatal("expected HasAppAuth() = false when AppInstallationID is zero") + } +} + +func TestHasAppAuthReturnsFalseWhenPrivateKeyFileMissing(t *testing.T) { + t.Parallel() + + repo := GenRepo{ + AppID: 12345, + AppInstallationID: 67890, + AppPrivateKeyFile: "", + } + + if repo.HasAppAuth() { + t.Fatal("expected HasAppAuth() = false when AppPrivateKeyFile is empty") + } +} + +func TestHasAppAuthReturnsFalseForEmptyRepo(t *testing.T) { + t.Parallel() + + if (GenRepo{}).HasAppAuth() { + t.Fatal("expected HasAppAuth() = false for zero-value GenRepo") + } +} + func TestSiteGetValuesScpStyle(t *testing.T) { t.Parallel() From 036e5f5236e3fea5406abb2543a21aba47201aa5 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Mon, 22 Jun 2026 09:41:08 +0200 Subject: [PATCH 14/44] fixed css for mobile ui --- webui/ui/index.html | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/webui/ui/index.html b/webui/ui/index.html index 3f8bf226..64bceb77 100644 --- a/webui/ui/index.html +++ b/webui/ui/index.html @@ -71,7 +71,15 @@ flex-shrink: 0; } .logo img { width: 32px; height: 32px; object-fit: contain; } - .nav-tabs { display: flex; align-items: stretch; height: 100%; } + .nav-tabs { + display: flex; + align-items: stretch; + height: 100%; + min-width: 0; + overflow-x: auto; + scrollbar-width: none; + } + .nav-tabs::-webkit-scrollbar { display: none; } .nav-tab { display: flex; align-items: center; @@ -98,9 +106,12 @@ display: flex; align-items: center; gap: 6px; + flex-shrink: 0; transition: color 0.15s, border-color 0.15s; } .theme-toggle:hover { color: var(--text); border-color: var(--text); } + #theme-icon { display: inline-flex; align-items: center; flex-shrink: 0; } + #theme-icon svg { display: block; } /* layout */ main { padding: 28px 24px; max-width: 1280px; margin: 0 auto; } .page { display: none; } @@ -410,6 +421,24 @@ ::-webkit-scrollbar-track { background: transparent; } ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; } ::-webkit-scrollbar-thumb:hover { background: var(--muted); } + + @media (max-width: 560px) { + nav { + padding: 0 12px; + gap: 12px; + } + .logo { gap: 8px; font-size: 15px; } + .logo img { width: 28px; height: 28px; } + .nav-tab { padding: 0 10px; font-size: 13px; } + .theme-toggle { + width: 34px; + height: 34px; + justify-content: center; + padding: 0; + } + #theme-label { display: none; } + main { padding: 20px 12px; } + } @@ -424,7 +453,7 @@ - From 029f4e9fb417f5ef36fc785bf2f2907db7fda53b Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Mon, 22 Jun 2026 09:45:16 +0200 Subject: [PATCH 15/44] extend ~ to home directory (#404) --- main.go | 36 ++++++++++++++++++++++++++++++--- main_test.go | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/main.go b/main.go index 2c828f2a..32685c1e 100644 --- a/main.go +++ b/main.go @@ -92,9 +92,7 @@ func readConfigFile(configfile string) []*types.Conf { continue } - for i, local := range c.Destination.Local { - c.Destination.Local[i].Path = substituteHomeForTildeInPath(local.Path) - } + expandConfigPaths(&c) if !reflect.ValueOf(c).IsZero() { if len(conf) > 0 { @@ -110,6 +108,38 @@ func readConfigFile(configfile string) []*types.Conf { return conf } +func expandConfigPaths(c *types.Conf) { + c.Log.FileLogging.Dir = substituteHomeForTildeInPath(c.Log.FileLogging.Dir) + + expandGenRepoPaths(c.Source.Gogs) + expandGenRepoPaths(c.Source.Gitlab) + expandGenRepoPaths(c.Source.Github) + expandGenRepoPaths(c.Source.Gitea) + expandGenRepoPaths(c.Source.BitBucket) + expandGenRepoPaths(c.Source.OneDev) + expandGenRepoPaths(c.Source.Sourcehut) + expandGenRepoPaths(c.Source.Any) + + expandGenRepoPaths(c.Destination.Gitlab) + expandGenRepoPaths(c.Destination.Github) + expandGenRepoPaths(c.Destination.Gitea) + expandGenRepoPaths(c.Destination.Gogs) + expandGenRepoPaths(c.Destination.OneDev) + expandGenRepoPaths(c.Destination.Sourcehut) + + for i, local := range c.Destination.Local { + c.Destination.Local[i].Path = substituteHomeForTildeInPath(local.Path) + } +} + +func expandGenRepoPaths(repos []types.GenRepo) { + for i := range repos { + repos[i].TokenFile = substituteHomeForTildeInPath(repos[i].TokenFile) + repos[i].SSHKey = substituteHomeForTildeInPath(repos[i].SSHKey) + repos[i].AppPrivateKeyFile = substituteHomeForTildeInPath(repos[i].AppPrivateKeyFile) + } +} + func getUserHome() (string, error) { usr, err := user.Current() if err != nil { diff --git a/main_test.go b/main_test.go index da2b02f1..b49d3ca1 100644 --- a/main_test.go +++ b/main_test.go @@ -84,6 +84,63 @@ destination: } } +func TestReadConfigFile_ExpandsHomeInFileBackedConfigKeys(t *testing.T) { + t.Parallel() + + config := `source: + github: + - token_file: "~/github-token" + sshkey: "~/github-ssh" + app_private_key_file: "~/github-app.pem" +destination: + gitea: + - token_file: "~/gitea-token" + sshkey: "~/gitea-ssh" + app_private_key_file: "~/gitea-app.pem" +log: + file-logging: + dir: "~/gickup-logs" + file: "gickup.log" +` + f, err := os.CreateTemp(t.TempDir(), "gickup-test-*.yml") + if err != nil { + t.Fatalf("create temp file: %v", err) + } + configPath := f.Name() + if _, err := f.WriteString(config); err != nil { + t.Fatalf("write config: %v", err) + } + f.Close() + + confs := readConfigFile(configPath) + if len(confs) != 1 { + t.Fatalf("expected 1 config, got %d", len(confs)) + } + + source := confs[0].Source.Github[0] + for name, path := range map[string]string{ + "source token_file": source.TokenFile, + "source sshkey": source.SSHKey, + "source app_private_key_file": source.AppPrivateKeyFile, + } { + if strings.HasPrefix(path, "~") { + t.Fatalf("%s was not expanded: %q", name, path) + } + } + + destination := confs[0].Destination.Gitea[0] + for name, path := range map[string]string{ + "destination token_file": destination.TokenFile, + "destination sshkey": destination.SSHKey, + "destination app_private_key_file": destination.AppPrivateKeyFile, + "log file-logging dir": confs[0].Log.FileLogging.Dir, + } { + if strings.HasPrefix(path, "~") { + t.Fatalf("%s was not expanded: %q", name, path) + } + } +} + func TestReadConfigFile_S3UseStaticCredsTrue(t *testing.T) { t.Parallel() From e585cdea3e88e95b746a8d40467839d0f4268f04 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Mon, 22 Jun 2026 10:59:37 +0200 Subject: [PATCH 16/44] updated documentation (#405) --- conf.example.yml | 10 +++++- gickup_spec.json | 80 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/conf.example.yml b/conf.example.yml index 4369a9f3..caac106e 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -29,6 +29,7 @@ source: wiki: true # includes wiki too issues: true # back up issues, works only locally starred: true # includes the user's starred repositories too + contributed: true # includes repositories the user contributed to filter: stars: 100 # only clone repos with 100 stars lastactivity: 1y # only clone repos which had activity during the last year @@ -150,9 +151,11 @@ source: excludeforks: true # exclude forked repositories bitbucket: - user: some-user # the user you want to clone the repositories from. - - token: some-token + token: some-token # token_file: token.txt # alternatively, specify token in a file url: http(s)://url-to-bitbucket # if empty, it uses https://bitbucket.org + organization: some-workspace # workspace to list repositories from + email: your-email@example.com # email is used for Bitbucket API basic auth username: your-user # user is used to clone the repo with password: your-password ssh: true # can be true or false @@ -245,6 +248,10 @@ destination: - token: some-token # token_file: token.txt # alternatively, specify token in a file url: http(s)://url-to-gitlab + user: some-name # can be a user or a group, it must exist on the system + visibility: + repositories: private # private, public, default: source repository visibility + force: false # force push to destination mirror: enabled: true # if set to true, gickup will clone the repository and push it to gitlab itself github: @@ -294,6 +301,7 @@ destination: accesskey: your-access-key # can be an environment variable, just don't add a $ in front of it secretkey: your-secret-key # can be an environment variable, just don't add a $ in front of it token: your-token # can be an environment variable, just don't add a $ in front of it + region: your-region # S3 region, for example us-east-1 zip: false # if true, will zip the entire git repo into a single zip file and upload that instead usessl: true # wheter to use ssl or not storageclass: "" # storage class for all repos to be uploaded to this S3 bucket. E.g. for AWS: STANDARD, STANDARD_IA, GLACIER, etc. diff --git a/gickup_spec.json b/gickup_spec.json index 9a50d8f9..e8777716 100644 --- a/gickup_spec.json +++ b/gickup_spec.json @@ -70,6 +70,9 @@ "gists": { "$ref": "#/definitions/source/properties/gists" }, + "contributed": { + "$ref": "#/definitions/source/properties/contributed" + }, "app_id": { "$ref": "#/definitions/source/properties/app_id" }, @@ -403,6 +406,12 @@ "user": { "$ref": "#/definitions/source/properties/user" }, + "organization": { + "$ref": "#/definitions/source/properties/organization" + }, + "email": { + "$ref": "#/definitions/source/properties/email" + }, "username": { "$ref": "#/definitions/source/properties/username" }, @@ -560,6 +569,10 @@ "type": "string", "description": "The token to authenticate against the S3 server" }, + "region": { + "type": "string", + "description": "The S3 region to use" + }, "zip": { "type": "boolean", "description": "If true, zip the entire Git repo into a single zip file and upload that instead" @@ -633,6 +646,9 @@ "url": { "$ref": "#/definitions/destination/properties/url" }, + "user": { + "$ref": "#/definitions/destination/properties/user" + }, "force": { "$ref": "#/definitions/destination/properties/force" }, @@ -644,6 +660,15 @@ } }, "additionalProperties": false + }, + "visibility": { + "type": "object", + "properties": { + "repositories": { + "$ref": "#/definitions/visibility/properties/repositories" + } + }, + "additionalProperties": false } }, "additionalProperties": false @@ -789,6 +814,51 @@ }, "additionalProperties": false } + }, + "azureblob": { + "type": "array", + "items": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The Azure Blob Storage endpoint URL, optionally including a SAS token" + }, + "container": { + "type": "string", + "description": "The Azure Blob Storage container to store the backups in" + }, + "useclicredential": { + "type": "boolean", + "description": "If true, use Azure CLI credentials to authenticate" + }, + "tenantid": { + "type": "string", + "description": "The Azure tenant ID for client secret authentication" + }, + "clientid": { + "type": "string", + "description": "The Azure app registration client ID" + }, + "clientsecret": { + "type": "string", + "description": "The Azure app registration client secret" + }, + "structured": { + "type": "boolean", + "description": "if set to `true`, it checks out the repos in a more structured way, like `hoster/user|organization/repository`" + }, + "zip": { + "type": "boolean", + "description": "If true, zip the entire Git repo into a single zip file and upload that instead" + }, + "datecreatedir": { + "type": "boolean", + "description": "If true, gickup will create a directory for backup with the creation date" + } + }, + "additionalProperties": false + } } } } @@ -823,6 +893,11 @@ "type": "string", "description": "The organization to clone the repositories from" }, + "email": { + "$id": "#/definitions/source/properties/email", + "type": "string", + "description": "The email address to authenticate against the source" + }, "username": { "$id": "#/definitions/source/properties/username", "type": "string", @@ -899,6 +974,11 @@ "type": "boolean", "description": "Include the gists in the backup" }, + "contributed": { + "$id": "#/definitions/source/properties/contributed", + "type": "boolean", + "description": "Include repositories the user contributed to" + }, "app_id": { "$id": "#/definitions/source/properties/app_id", "type": "integer", From 7dc1fa0d72a991d3c0686af0bc4d13140932c558 Mon Sep 17 00:00:00 2001 From: Jonathan Van Eenwyk Date: Wed, 8 Jul 2026 11:28:58 +0530 Subject: [PATCH 17/44] fix: gitlab organization filtering not including nested subgroups (#409) Gitlab supports nesting subgroups up to 20 levels deep. However, these two filters were actually applied to projects in the provided organization/group as well as projects in any immediate subgroup, but not to projects in more deeply nested subgroups. Presumably, the original intention of these filters was that the entire organization/group would be either included or excluded, not only projects in the group and its immediate subgroups. Fix by using ListDescendantGroups() to get all descendant/nested groups. Also, use gitlab.ScanAndCollect() to handle paginated results. --- gitlab/gitlab.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index e74c7d0f..ff78c514 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -209,7 +209,11 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Msg(err.Error()) continue } - subgroups, _, err := client.Groups.ListSubGroups(group.ID, &gitlab.ListSubGroupsOptions{}) + + subgroups, err := gitlab.ScanAndCollect(func(p gitlab.PaginationOptionFunc) ([]*gitlab.Group, *gitlab.Response, error) { + return client.Groups.ListDescendantGroups(group.ID, &gitlab.ListDescendantGroupsOptions{}, p) + }) + for _, sub := range subgroups { includeorgs[sub.FullPath] = true } @@ -222,7 +226,11 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Msg(err.Error()) continue } - subgroups, _, err := client.Groups.ListSubGroups(group.ID, &gitlab.ListSubGroupsOptions{}) + + subgroups, err := gitlab.ScanAndCollect(func(p gitlab.PaginationOptionFunc) ([]*gitlab.Group, *gitlab.Response, error) { + return client.Groups.ListDescendantGroups(group.ID, &gitlab.ListDescendantGroupsOptions{}, p) + }) + for _, sub := range subgroups { excludeorgs[sub.FullPath] = true } From 88d9c3981f10758a74ca4fb1b76e112f49d94c5d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:18:51 +0200 Subject: [PATCH 18/44] Bump golang.org/x/crypto from 0.51.0 to 0.52.0 (#410) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.51.0 to 0.52.0. - [Commits](https://github.com/golang/crypto/compare/v0.51.0...v0.52.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.52.0 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 +--- go.sum | 25 ++++--------------------- 2 files changed, 5 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 1a14a5da..e89b9171 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/rs/zerolog v1.35.1 github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed gitlab.com/gitlab-org/api/client-go v1.46.0 - golang.org/x/crypto v0.51.0 + golang.org/x/crypto v0.52.0 golang.org/x/oauth2 v0.36.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -86,13 +86,11 @@ require ( github.com/tinylib/msgp v1.6.4 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - github.com/zeebo/xxh3 v1.1.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/protobuf v1.36.11 // indirect - gopkg.in/ini.v1 v1.67.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index fa207204..b10fbcdb 100644 --- a/go.sum +++ b/go.sum @@ -60,8 +60,8 @@ github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= -github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= -github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= @@ -147,16 +147,12 @@ github.com/mattn/go-isatty v0.0.22 h1:j8l17JJ9i6VGPUFUYoTUKPSgKe/83EYU2zBC7YNKMw github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= github.com/melbahja/goph v1.5.0 h1:RQUBpLvfg3i7fjfG8rTcSWyMjVRfdhwrrfQhjYee4dQ= github.com/melbahja/goph v1.5.0/go.mod h1:dDwo+44cmvfDLdiVpc6fJxexf5BA5yEDUeE5YgtuDO4= -github.com/melbahja/goph v1.5.1 h1:gdZyjn5nM6FO6mAZrv7XqE2gArQbb+V6FdIyH3Ru3vA= -github.com/melbahja/goph v1.5.1/go.mod h1:dDwo+44cmvfDLdiVpc6fJxexf5BA5yEDUeE5YgtuDO4= github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= github.com/minio/minio-go/v7 v7.0.100 h1:ShkWi8Tyj9RtU57OQB2HIXKz4bFgtVib0bbT1sbtLI8= github.com/minio/minio-go/v7 v7.0.100/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw= -github.com/minio/minio-go/v7 v7.2.0 h1:RCJM0R1XOsRs+A3x3UCaf3ZYbByDaLjFeAi+YCQEPhs= -github.com/minio/minio-go/v7 v7.2.0/go.mod h1:EU9hENAStx/xXduNdrGO5e4X5vk19NtgB+RIPjZO8o0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -201,14 +197,8 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/skeema/knownhosts v1.3.2 h1:EDL9mgf4NzwMXCTfaxSD/o/a5fxDw/xL9nkU28JjdBg= github.com/skeema/knownhosts v1.3.2/go.mod h1:bEg3iQAuw+jyiw+484wwFJoKSLwcfd7fqRy+N0QTiow= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= @@ -217,10 +207,6 @@ github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= -github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= -github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= -github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= -github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= gitlab.com/gitlab-org/api/client-go v1.46.0 h1:YxBWFZIFYKcGESCb9fpkwzouo+apyB9pr/XTWzNoL24= gitlab.com/gitlab-org/api/client-go v1.46.0/go.mod h1:FtgyU6g2HS5+fMhw6nLK96GBEEBx5MzntOiJWfIaiN8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -233,8 +219,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= -golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -272,14 +258,11 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/ini.v1 v1.67.2 h1:JtOSMb9OuaCZKr7h5D/h6iii14sK0hLbplTc6frx4Ss= -gopkg.in/ini.v1 v1.67.2/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From dfee42bf30ef84e7201ef3a72760476f91d94af6 Mon Sep 17 00:00:00 2001 From: Michael Herberth <82085656+MichaelHerberth@users.noreply.github.com> Date: Fri, 24 Jul 2026 20:54:45 +0200 Subject: [PATCH 19/44] Add configurable GitHub Enterprise URL support (#411) Co-authored-by: Michael Herberth --- conf.example.yml | 1 + github/github.go | 103 ++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 86 insertions(+), 18 deletions(-) diff --git a/conf.example.yml b/conf.example.yml index caac106e..bd71fc89 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -7,6 +7,7 @@ source: # token_file: token.txt user: some-user # the user you want to clone the repositories from. # if you want to get everything from your user, leave out the user parameter and just use the token. + # url: https://example.ghe.com # optional: GitHub Enterprise Server base URL; defaults to https://github.com # for the clone process, either use: # - username + password # - sshkey diff --git a/github/github.go b/github/github.go index cad93685..8e983a39 100644 --- a/github/github.go +++ b/github/github.go @@ -48,17 +48,56 @@ type V4Repo struct { var sub zerolog.Logger +// githubInstanceURL returns the canonical base URL for a GitHub instance. +// Defaults to "https://github.com" if rawURL is empty. +func githubInstanceURL(rawURL string) string { + if rawURL == "" { + return "https://github.com" + } + return strings.TrimRight(rawURL, "/") +} + +// isGHE returns true when the URL refers to a GitHub Enterprise Server instance. +func isGHE(instanceURL string) bool { + u := strings.TrimPrefix(instanceURL, "https://") + u = strings.TrimPrefix(u, "http://") + return !strings.HasPrefix(u, "github.com") +} + +// hosterFromURL extracts the hostname portion from a GitHub instance URL. +func hosterFromURL(instanceURL string) string { + u := strings.TrimPrefix(instanceURL, "https://") + u = strings.TrimPrefix(u, "http://") + if idx := strings.Index(u, "/"); idx != -1 { + return u[:idx] + } + return u +} + // newGithubClient creates an authenticated GitHub client. // When App authentication fields are configured it uses a GitHub App installation token; // otherwise it falls back to a personal access token or an unauthenticated client. // It returns the client, the access token string (for git operations), and any error. func newGithubClient(ctx context.Context, repo types.GenRepo) (*github.Client, string, error) { + instURL := githubInstanceURL(repo.URL) + if repo.HasAppAuth() { itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, repo.AppID, repo.AppInstallationID, repo.AppPrivateKeyFile) if err != nil { return nil, "", fmt.Errorf("github app auth: %w", err) } - client := github.NewClient(&http.Client{Transport: itr}) + var client *github.Client + if isGHE(instURL) { + apiBase := instURL + "/api/v3/" + uploadURL := instURL + "/uploads/" + itr.BaseURL = apiBase + client, err = github.NewEnterpriseClient(apiBase, uploadURL, &http.Client{Transport: itr}) + if err != nil { + return nil, "", fmt.Errorf("github enterprise client: %w", err) + } + } else { + client = github.NewClient(&http.Client{Transport: itr}) + } token, err := itr.Token(ctx) if err != nil { return nil, "", fmt.Errorf("github app token: %w", err) @@ -67,22 +106,44 @@ func newGithubClient(ctx context.Context, repo types.GenRepo) (*github.Client, s } token := repo.GetToken() + + var tc *http.Client + if token != "" { + ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) + tc = oauth2.NewClient(ctx, ts) + } + + if isGHE(instURL) { + apiBase := instURL + "/api/v3/" + uploadURL := instURL + "/uploads/" + client, err := github.NewEnterpriseClient(apiBase, uploadURL, tc) + if err != nil { + return nil, "", fmt.Errorf("github enterprise client: %w", err) + } + return client, token, nil + } + if token == "" { return github.NewClient(nil), "", nil } - ts := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: token}) - tc := oauth2.NewClient(ctx, ts) return github.NewClient(tc), token, nil } -func getv4(token, user string) []V4Repo { +func getv4(token, user, instanceURL string) []V4Repo { repos := []V4Repo{} tokenSource := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, ) oauth2Client := oauth2.NewClient(context.Background(), tokenSource) - client := githubv4.NewClient(oauth2Client) + + var client *githubv4.Client + if isGHE(instanceURL) { + graphqlURL := instanceURL + "/api/graphql" + client = githubv4.NewEnterpriseClient(graphqlURL, oauth2Client) + } else { + client = githubv4.NewClient(oauth2Client) + } var query Query variables := map[string]interface{}{ @@ -110,7 +171,7 @@ func getv4(token, user string) []V4Repo { return repos } -func addWiki(r github.Repository, repo types.GenRepo, token string) types.Repo { +func addWiki(r github.Repository, repo types.GenRepo, token, hoster string) types.Repo { if !(r.GetHasWiki() && repo.Wiki && types.StatRemote(r.GetCloneURL(), r.GetSSHURL(), repo)) { return types.Repo{} @@ -123,7 +184,7 @@ func addWiki(r github.Repository, repo types.GenRepo, token string) types.Repo { Token: token, Origin: repo, Owner: r.GetOwner().GetLogin(), - Hoster: "github.com", + Hoster: hoster, Description: r.GetDescription(), Private: r.GetPrivate(), } @@ -134,7 +195,9 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { ran := false repos := []types.Repo{} for _, repo := range conf.Source.Github { - sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com") + instURL := githubInstanceURL(repo.URL) + hoster := hosterFromURL(instURL) + sub = logger.CreateSubLogger("stage", "github", "url", instURL) err := repo.Filter.ParseDuration() if err != nil { sub.Warn(). @@ -183,7 +246,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if repo.HasAppAuth() { sub.Warn().Msg("contributed repos are not supported with GitHub App authentication, skipping") } else { - for _, r := range getv4(token, v4user) { + for _, r := range getv4(token, v4user, instURL) { githubRepo, _, err := client.Repositories.Get(context.Background(), r.User, r.Repository) if err != nil { sub.Error(). @@ -321,13 +384,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Token: token, Origin: repo, Owner: r.GetOwner().GetLogin(), - Hoster: "github.com", + Hoster: hoster, Description: r.GetDescription(), Private: r.GetPrivate(), Issues: GetIssues(r, client, repo), NoTokenUser: true, }) - wiki := addWiki(*r, repo, token) + wiki := addWiki(*r, repo, token, hoster) if wiki.Name != "" { repos = append(repos, wiki) } @@ -345,13 +408,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Token: token, Origin: repo, Owner: r.GetOwner().GetLogin(), - Hoster: "github.com", + Hoster: hoster, Description: r.GetDescription(), Private: r.GetPrivate(), Issues: GetIssues(r, client, repo), NoTokenUser: true, }) - wiki := addWiki(*r, repo, token) + wiki := addWiki(*r, repo, token, hoster) if wiki.Name != "" { repos = append(repos, wiki) } @@ -364,13 +427,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { Token: token, Origin: repo, Owner: r.GetOwner().GetLogin(), - Hoster: "github.com", + Hoster: hoster, Description: r.GetDescription(), Private: r.GetPrivate(), Issues: GetIssues(r, client, repo), NoTokenUser: true, }) - wiki := addWiki(*r, repo, token) + wiki := addWiki(*r, repo, token, hoster) if wiki.Name != "" { repos = append(repos, wiki) } @@ -394,14 +457,18 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { for _, gist := range gists { sub.Debug().Msg(gist.GetHTMLURL()) + gistSSHURL := fmt.Sprintf("git@gist.github.com:%s.git", gist.GetID()) + if isGHE(instURL) { + gistSSHURL = fmt.Sprintf("git@%s:gist/%s.git", hoster, gist.GetID()) + } repos = append(repos, types.Repo{ Name: fmt.Sprintf("gists%c%s", os.PathSeparator, gist.GetID()), URL: gist.GetHTMLURL(), - SSHURL: fmt.Sprintf("git@gist.github.com:%s.git", gist.GetID()), + SSHURL: gistSSHURL, Token: token, Origin: repo, Owner: gist.GetOwner().GetLogin(), - Hoster: "github.com", + Hoster: hoster, Description: gist.GetDescription(), Private: !gist.GetPublic(), NoTokenUser: true, @@ -418,7 +485,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { // GetOrCreate Get or create a repository func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { - sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com") + sub = logger.CreateSubLogger("stage", "github", "url", githubInstanceURL(destination.URL)) client, _, err := newGithubClient(context.TODO(), destination) if err != nil { return "", err From 66eaf7bd2cc80fb9cb13f113c3a17d7c25e42baf Mon Sep 17 00:00:00 2001 From: Yorgos Saslis <2420882+gsaslis@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:41:13 +0200 Subject: [PATCH 20/44] Adds Radicle destination (#413) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Adds Radicle destination This destination allows mirroring repositories into the storage of a local [Radicle](https://radicle.network) node. Requires `git` + Radicle Command Line Interface (CLI) tools installed, i.e. `rad` and `git-remote-rad` binaries. See https://radicle.network/install . The repository is initialized with `rad init` on the first run and pushed through the `git-remote-rad` helper afterwards, so the refs are signed with the key of the Radicle profile. Because every backup run works on a fresh temporary clone, each run has to find the Repository ID (RID) created for an upstream on an earlier run: every mirror records its upstream URL in a "dev.gickup" extension of the signed Radicle identity document `payload`, and the storage is scanned for a repository whose recorded source matches and whose refs are signed by this profile's key — comparable to the existence lookups the forge destinations make against their APIs, just against local storage. (Local-first, For The Win!) Unlike other forge destinations we cannot use go-git for pushing, because Radicle repositories are only reachable through the `git-remote-rad` remote helper, which go-git does not support. All storage-touching operations therefore shell out to the system `git` and `rad` binaries, so identity creation and signed-refs handling stay inside the Radicle tooling. Signed-off-by: Yorgos Saslis * fixup! Adds Radicle destination Signed-off-by: Yorgos Saslis * fixup! Adds Radicle destination Signed-off-by: Yorgos Saslis --------- Signed-off-by: Yorgos Saslis Co-authored-by: Andreas Wachter --- .golangci.yml | 3 + README.md | 1 + conf.example.yml | 8 + gickup_spec.json | 31 +++ main.go | 64 ++++++ radicle/radicle.go | 429 ++++++++++++++++++++++++++++++++++++++++ radicle/radicle_test.go | 237 ++++++++++++++++++++++ types/types.go | 16 +- 8 files changed, 788 insertions(+), 1 deletion(-) create mode 100644 radicle/radicle.go create mode 100644 radicle/radicle_test.go diff --git a/.golangci.yml b/.golangci.yml index 2f307f1f..b6b8f782 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -67,6 +67,9 @@ linters: - linters: [gosec] path: github/github_test.go text: "G101:" + - linters: [gosec] + path: radicle/radicle_test.go + text: "G101:" # Global exclusions for staticcheck style suggestions - linters: [staticcheck] text: "SA1019:" diff --git a/README.md b/README.md index 6daa7a1b..2b9c8b74 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ You can clone/mirror repositories to: - Gogs - OneDev - Sourcehut +- Radicle - Local - S3 diff --git a/conf.example.yml b/conf.example.yml index bd71fc89..ca3cfc19 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -294,6 +294,14 @@ destination: bare: true # clone the repositories as bare mirror: true # create mirror clones lfs: true # clone lfs repos, !! ATTENTION !! git and git-lfs must be installed on the system! + radicle: + # rad reads the environment itself: RAD_HOME selects which radicle profile to + # mirror into (defaults to ~/.radicle), and RAD_PASSPHRASE unlocks its key if + # it has a passphrase. Export them before running gickup. + - force: true # if set to true, refs that diverged from upstream are overwritten (non-fast-forward updates) + prune: true # if set to true, refs that no longer exist upstream are deleted from the mirror + visibility: source # public, private or source, source follows the visibility of the source repository. default: source + issues: false # [COMING SOON] recreate the source repo's issues as radicle issues (the source must also have issues: true). s3: - endpoint: somewhere:9000 # whatever your s3 endpoint is structured: true # checks repos out like hostersite/user|organization/repo diff --git a/gickup_spec.json b/gickup_spec.json index e8777716..3ba70423 100644 --- a/gickup_spec.json +++ b/gickup_spec.json @@ -859,6 +859,37 @@ }, "additionalProperties": false } + }, + "radicle": { + "type": "array", + "description": "Mirror repositories into the storage of a local Radicle node. Requires the `git`, `rad` and `git-remote-rad` binaries.", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "force": { + "type": "boolean", + "description": "overwrite refs on the mirror that have diverged from upstream (non-fast-forward updates)" + }, + "prune": { + "type": "boolean", + "description": "delete refs on the mirror that no longer exist upstream" + }, + "visibility": { + "type": "string", + "enum": [ + "public", + "private", + "source" + ], + "description": "visibility of the mirrored repository, `source` follows the source repository. default: source" + }, + "issues": { + "type": "boolean", + "description": "[COMING SOON] recreate the source repository's issues." + } + } + } } } } diff --git a/main.go b/main.go index 32685c1e..09be5997 100644 --- a/main.go +++ b/main.go @@ -28,6 +28,7 @@ import ( "github.com/cooperspencer/gickup/metrics/ntfy" "github.com/cooperspencer/gickup/metrics/prometheus" "github.com/cooperspencer/gickup/onedev" + "github.com/cooperspencer/gickup/radicle" "github.com/cooperspencer/gickup/s3" "github.com/cooperspencer/gickup/sourcehut" "github.com/cooperspencer/gickup/types" @@ -912,6 +913,69 @@ func backup(repos []types.Repo, conf *types.Conf) { } } + for _, d := range conf.Destination.Radicle { + repotime := time.Now() + status := 0 + + radhome, _ := radicle.Home() + + log.Info(). + Str("stage", "radicle"). + Str("home", radhome). + Msgf("mirroring %s to %s", types.Blue(r.Name), radhome) + + if !cli.Dry { + func() { + tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("radicle-%x", repotime)) + if err != nil { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Msg(err.Error()) + return + } + + defer os.RemoveAll(tempdir) + _, err = local.TempClone(r, tempdir) + if err != nil { + if errors.Is(err, git.NoErrAlreadyUpToDate) { + log.Info(). + Str("stage", "radicle"). + Str("url", r.URL). + Msg(err.Error()) + } else { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Str("git", "clone"). + Msg(err.Error()) + return + } + } + + rid, err := radicle.Mirror(r, d, tempdir) + if err != nil { + log.Error(). + Str("stage", "radicle"). + Str("url", r.URL). + Msg(err.Error()) + return + } + + log.Info(). + Str("stage", "radicle"). + Str("rid", fmt.Sprintf("rad:%s", rid)). + Msgf("mirrored %s", types.Green(r.Name)) + + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "radicle", radhome).Set(time.Since(repotime).Seconds()) + status = 1 + }() + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "radicle", radhome).Set(float64(status)) + prometheus.DestinationBackupsComplete.WithLabelValues("radicle").Inc() + } + } + prometheus.SourceBackupsComplete.WithLabelValues(r.Name).Inc() } } diff --git a/radicle/radicle.go b/radicle/radicle.go new file mode 100644 index 00000000..2ba351ac --- /dev/null +++ b/radicle/radicle.go @@ -0,0 +1,429 @@ +// Package radicle mirrors repositories into the local storage of a Radicle +// node. +package radicle + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/cooperspencer/gickup/logger" + "github.com/cooperspencer/gickup/types" + "github.com/rs/zerolog" +) + +var sub zerolog.Logger + +// payloadCache memoizes the storage scan so scanStorage inspects the storage +// only once per RAD_HOME, instead of once per mirrored repository. It is +// populated on the first scan and kept in sync as initRepo creates mirrors, +// so this process never re-inspects storage it already knows. +var payloadCache = map[string]map[string]payload{} + +// payload is the identity document `payload` section, as returned by `rad inspect --payload`. +type payload struct { + Project struct { + Name string `json:"name"` + Description string `json:"description"` + DefaultBranch string `json:"defaultBranch"` + } `json:"xyz.radicle.project"` + // Gickup is the provenance extension this package adds to the identity + // document of every mirror it initializes. + Gickup struct { + Source string `json:"source"` + } `json:"dev.gickup"` +} + +// Mirror pushes the repository cloned at tempdir into the Radicle storage +// configured by d, initializing it with `rad init` on first sight. It returns +// the RID of the mirror. +func Mirror(repo types.Repo, dest types.Radicle, tempdir string) (string, error) { + home, _ := Home() + sub = logger.CreateSubLogger("stage", "radicle", "home", home) + + if err := probe(); err != nil { + return "", err + } + + // rad and git-remote-rad read RAD_HOME and RAD_PASSPHRASE from the + // environment themselves, so the ambient environment is all they need. + env := os.Environ() + + nid, err := nodeID(env) + if err != nil { + return "", err + } + + rid, err := scanStorage(repo, nid, env) + if err != nil { + return "", err + } + + if rid == "" { + rid, err = initRepo(repo, dest, tempdir, env) + if err != nil { + return "", err + } + + sub.Info(). + Str("rid", fmt.Sprintf("rad:%s", rid)). + Msgf("initialized %s", types.Green(repo.Name)) + } else if dest.Force || dest.Prune { + if err := fetchMirrorState(tempdir, rid, nid, env); err != nil { + return "", err + } + } + + if err := push(dest, tempdir, rid, nid, env); err != nil { + return "", err + } + + // Issues coming soon + if dest.Issues { + sub.Warn().Msg("Issue migration to Radicle not yet supported.") + } + + return rid, nil +} + +// probe checks that all binaries required for mirroring to Radicle are available. +func probe() error { + for _, bin := range []string{"git", "rad", "git-remote-rad"} { + if _, err := exec.LookPath(bin); err != nil { + return fmt.Errorf("%s is not installed", bin) + } + } + + return nil +} + +// fetchMirrorState fetches the current refs of the mirror into the temporary +// clone. When upstream history was rewritten or a branch was deleted, the old +// ref targets no longer exist in a fresh clone, and the remote helper then +// fails to re-sign rad/sigrefs after the ref update, leaving the signed refs +// pointing at the old state. Fetching first keeps the old objects resolvable +// so refs and signature update together. Pushes without force and prune are +// fast-forward only, where the old targets are always present, so the fetch +// is skipped. +func fetchMirrorState(tempdir, rid, nid string, env []string) error { + _, err := run(tempdir, env, "git", "fetch", "--no-tags", + fmt.Sprintf("rad://%s/%s", rid, nid), + "+refs/heads/*:refs/gickup/heads/*", + "+refs/tags/*:refs/gickup/tags/*") + + return err +} + +// Home returns the profile's Radicle home as reported by `rad self --home`. +// Propagates error accordingly, so it can be handled at call sites. +func Home() (string, error) { + return run("", os.Environ(), "rad", "self", "--home") +} + +// run executes name with args in dir, returning trimmed stdout. Stderr is +// folded into the returned error. +func run(dir string, env []string, name string, args ...string) (string, error) { + cmd := exec.CommandContext(context.Background(), name, args...) + cmd.Dir = dir + cmd.Env = env + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + msg := strings.TrimSpace(stderr.String()) + if msg == "" { + msg = strings.TrimSpace(stdout.String()) + } + + return "", fmt.Errorf("%s %s: %w: %s", name, strings.Join(args, " "), err, msg) + } + + return strings.TrimSpace(stdout.String()), nil +} + +// nodeID returns the node ID of the configured profile, which namespaces the +// push URL. `rad self --nid` is deprecated but still works without a running +// node, so it is used as the fallback. +func nodeID(env []string) (string, error) { + nid, err := run("", env, "rad", "node", "status", "--only", "nid") + if err != nil { + nid, err = run("", env, "rad", "self", "--nid") + if err != nil { + return "", err + } + } + + if nid == "" { + return "", fmt.Errorf("couldn't determine the node id of the radicle profile") + } + + return nid, nil +} + +// marker returns the human-readable provenance note that initRepo appends to +// the mirror's description. Any credentials embedded in the URL are stripped, because +// the description becomes part of the signed identity payload that is +// replicated to the network. +func marker(upstream string) string { + return fmt.Sprintf("(mirror of %s)", stripCredentials(upstream)) +} + +// stripCredentials removes the userinfo part from a URL, so tokens or +// passwords configured inline (https://user:token@host/...) never leave the +// local configuration. Non-URL remotes (e.g. scp-style ssh) pass through. This +// is useful because the identity document, where this URL is stored, is signed +// and published to the network. +func stripCredentials(upstream string) string { + if !strings.Contains(upstream, "://") { + return upstream + } + + parsed, err := url.Parse(upstream) + if err != nil || parsed.User == nil { + return upstream + } + + parsed.User = nil + + return parsed.String() +} + +// scanStorage searches the Radicle storage for the repository that mirrors +// the upstream URL. A candidate must record the URL as its source in the +// dev.gickup `payload` of its identity document, and list this profile among +// that identity's delegates. The second check is what makes adoption +// unforgeable: the source claim lives in the identity document, whose contents +// are only authoritative because its delegates signed them. +func scanStorage(repo types.Repo, nid string, env []string) (string, error) { + inspected, err := payloadsFor(env) + if err != nil { + return "", err + } + + matches := []string{} + source := stripCredentials(repo.URL) + + for rid, p := range inspected { + if p.Gickup.Source == source && isDelegate(rid, nid, env) { + matches = append(matches, rid) + } + } + + switch len(matches) { + case 0: + return "", nil + case 1: + return matches[0], nil + default: + return "", fmt.Errorf( + "found %d repositories in storage mirroring %s - remove the duplicates so only one remains", + len(matches), repo.URL) + } +} + +// isDelegate reports whether this profile's key (nid) is a delegate of the +// stored repository's identity. We consider this enough evidence that the +// information we read from the identity doc can be trusted. +func isDelegate(rid, nid string, env []string) bool { + out, err := run("", env, "rad", "inspect", fmt.Sprintf("rad:%s", rid), "--delegates") + if err != nil { + return false + } + + // Each line looks like "did:key: (alias)". + did := "did:key:" + nid + for _, line := range strings.Split(out, "\n") { + if fields := strings.Fields(line); len(fields) > 0 && fields[0] == did { + return true + } + } + + return false +} + +// payloadsFor returns the identity payloads of all repositories in the Radicle +// profile storage. The storage is inspected the first time a given RAD_HOME is asked +// about and the result is cached, so the repeated scanStorage calls across a +// run's repositories don't each re-inspect it. +func payloadsFor(env []string) (map[string]payload, error) { + home, err := Home() + if err != nil { + return nil, err + } + + if cached, ok := payloadCache[home]; ok { + return cached, nil + } + + entries, err := os.ReadDir(filepath.Join(home, "storage")) + if err != nil { + if os.IsNotExist(err) { + payloadCache[home] = map[string]payload{} + return payloadCache[home], nil + } + + return nil, err + } + + current := map[string]payload{} + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + out, err := run("", env, "rad", "inspect", fmt.Sprintf("rad:%s", entry.Name()), "--payload") + if err != nil { + sub.Debug(). + Str("rid", fmt.Sprintf("rad:%s", entry.Name())). + Msg(err.Error()) + + continue + } + + var p payload + if err := json.Unmarshal([]byte(out), &p); err != nil { + continue + } + + current[entry.Name()] = p + } + + payloadCache[home] = current + + return current, nil +} + +// initRepo runs `rad init` on the temporary clone, creating the repository +// identity in storage, and returns the new RID. The upstream URL is recorded +// in a dev.gickup extension of the identity document `payload`, which is what +// scanStorage matches on, on the next run. +func initRepo(repo types.Repo, dest types.Radicle, tempdir string, env []string) (string, error) { + branch, err := run(tempdir, env, "git", "symbolic-ref", "--short", "HEAD") + if err != nil { + return "", err + } + + description := strings.TrimSpace(fmt.Sprintf("%s %s", repo.Description, marker(repo.URL))) + + visibility, err := visibilityFlag(dest, repo) + if err != nil { + return "", err + } + + _, err = run(tempdir, env, "rad", "init", + fmt.Sprintf("--name=%s", repo.Name), + fmt.Sprintf("--description=%s", description), + fmt.Sprintf("--default-branch=%s", branch), + visibility, + "--no-confirm") + if err != nil { + return "", err + } + + rid, err := run(tempdir, env, "rad", ".") + if err != nil { + return "", err + } + + rid = strings.TrimPrefix(rid, "rad:") + + source, err := json.Marshal(stripCredentials(repo.URL)) + if err != nil { + return rid, err + } + + if err := recordMirrorSource(tempdir, env, source, rid); err != nil { + return rid, err + } + + updatePayloadCache(repo, rid) + + return rid, nil +} + +// updatePayloadCache Remember the new mirror so a later run in this process finds it without +// re-inspecting storage — and never tries to initialize it a second time. +func updatePayloadCache(repo types.Repo, rid string) { + home, err := Home() + if err != nil { + return + } + if payloadCache[home] == nil { + payloadCache[home] = map[string]payload{} + } + var newPayload payload + newPayload.Gickup.Source = stripCredentials(repo.URL) + payloadCache[home][rid] = newPayload +} + +// recordMirrorSource Captures the upstream source, so an existing mirror can be reused, +// avoiding duplicates. A failure here fails the overall +// backup, and the error names the repository because the storage offers +// no rollback for a created identity: the operator has to record the +// source or remove the repository by hand. +func recordMirrorSource(tempdir string, env []string, source []byte, rid string) error { + if _, err := run(tempdir, env, "rad", "id", "update", + "--payload", "dev.gickup", "source", string(source), + "--title", "Record mirror source", + "--no-confirm"); err != nil { + return fmt.Errorf( + "repository rad:%s was created but recording its mirror source failed - record dev.gickup.source manually or remove the repository: %w", + rid, err) + } + return nil +} + +// visibilityFlag maps the configured visibility to the rad init flag. +func visibilityFlag(dest types.Radicle, repo types.Repo) (string, error) { + switch dest.Visibility { + case "public": + return "--public", nil + case "private": + return "--private", nil + case "", "source": + if repo.Private { + return "--private", nil + } + + return "--public", nil + default: + return "", fmt.Errorf("invalid radicle visibility %s, must be public, private or source", dest.Visibility) + } +} + +// pushArgs builds the git arguments that mirror all branches and tags into +// the profile's namespace of the stored repository. +func pushArgs(dest types.Radicle, rid, nid string) []string { + prefix := "" + if dest.Force { + prefix = "+" + } + + args := []string{"push"} + if dest.Prune { + args = append(args, "--prune") + } + + return append(args, + fmt.Sprintf("rad://%s/%s", rid, nid), + fmt.Sprintf("%srefs/heads/*:refs/heads/*", prefix), + fmt.Sprintf("%srefs/tags/*:refs/tags/*", prefix)) +} + +// push mirrors all branches and tags of the temporary clone into the profile's +// namespace of the stored repository. The remote helper updates and signs +// rad/sigrefs and announces to the node, so nothing else has to touch storage. +func push(dest types.Radicle, tempdir, rid, nid string, env []string) error { + _, err := run(tempdir, env, "git", pushArgs(dest, rid, nid)...) + + return err +} diff --git a/radicle/radicle_test.go b/radicle/radicle_test.go new file mode 100644 index 00000000..6f1464fc --- /dev/null +++ b/radicle/radicle_test.go @@ -0,0 +1,237 @@ +package radicle + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cooperspencer/gickup/types" +) + +func TestHome(t *testing.T) { + // Home delegates to `rad self --home`; stub rad and check we return what + // it reports, verbatim. + bin := t.TempDir() + script := "#!/bin/sh\n[ \"$1 $2\" = \"self --home\" ] && echo /stub/radhome\n" + if err := os.WriteFile(filepath.Join(bin, "rad"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", fmt.Sprintf("%s%c%s", bin, os.PathListSeparator, os.Getenv("PATH"))) + + home, err := Home() + if err != nil { + t.Fatalf("Home() error: %v", err) + } + if home != "/stub/radhome" { + t.Errorf("expected /stub/radhome from rad self --home, got %q", home) + } +} + +func TestVisibilityFlag(t *testing.T) { + t.Parallel() + + private := types.Repo{Private: true} + public := types.Repo{Private: false} + + for _, tc := range []struct { + visibility string + repo types.Repo + expected string + }{ + {"public", private, "--public"}, + {"private", public, "--private"}, + {"", private, "--private"}, + {"", public, "--public"}, + {"source", private, "--private"}, + {"source", public, "--public"}, + } { + flag, err := visibilityFlag(types.Radicle{Visibility: tc.visibility}, tc.repo) + if err != nil { + t.Errorf("visibility %q: %v", tc.visibility, err) + } + + if flag != tc.expected { + t.Errorf("visibility %q private=%v: expected %s, got %s", tc.visibility, tc.repo.Private, tc.expected, flag) + } + } + + if _, err := visibilityFlag(types.Radicle{Visibility: "invisible"}, public); err == nil { + t.Error("expected an error for an invalid visibility") + } +} + +func TestPushArgs(t *testing.T) { + t.Parallel() + + args := pushArgs(types.Radicle{}, "zRid", "zNid") + expected := "push rad://zRid/zNid refs/heads/*:refs/heads/* refs/tags/*:refs/tags/*" + + if strings.Join(args, " ") != expected { + t.Errorf("expected %q, got %q", expected, strings.Join(args, " ")) + } + + args = pushArgs(types.Radicle{Force: true, Prune: true}, "zRid", "zNid") + expected = "push --prune rad://zRid/zNid +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*" + + if strings.Join(args, " ") != expected { + t.Errorf("expected %q, got %q", expected, strings.Join(args, " ")) + } +} + +const testNID = "zTestNodeId" + +type fakePayload struct { + name string + // source is the upstream URL in the dev.gickup payload; empty simulates + // a repository that was not created by gickup. + source string + // owned repos list this profile among the identity delegates; unowned ones + // list a foreign delegate, simulating a repo merely seeded from the network. + owned bool +} + +// fakeRad puts a rad stub on PATH answering `rad self --home`, `rad inspect +// --payload` and `rad inspect --delegates` per configured rid, and +// creates an (empty) storage directory per rid so payloadsFor can enumerate +// them — letting scanStorage be tested without a radicle installation. +func fakeRad(t *testing.T, radhome string, repos map[string]fakePayload) { + t.Helper() + + bin := t.TempDir() + ourDID := "did:key:" + testNID + + var payloadCases, delegateCases string + for rid, p := range repos { + gickup := "" + if p.source != "" { + gickup = fmt.Sprintf("\"dev.gickup\": {\"source\": \"%s\"}, ", p.source) + } + payloadCases += fmt.Sprintf("rad:%s) echo '{%s\"xyz.radicle.project\": {\"name\": \"%s\", \"defaultBranch\": \"main\"}}' ;;\n", rid, gickup, p.name) + + delegate := "did:key:zForeignNode" + if p.owned { + delegate = ourDID + } + delegateCases += fmt.Sprintf("rad:%s) echo '%s (alias)' ;;\n", rid, delegate) + } + + script := "#!/bin/sh\n" + + "if [ \"$1 $2\" = \"self --home\" ]; then echo \"$RAD_HOME\"; exit 0; fi\n" + + "case \"$3\" in\n" + + "--payload) case \"$2\" in\n" + payloadCases + "*) echo 'unknown repository' >&2; exit 1 ;;\nesac ;;\n" + + "--delegates) case \"$2\" in\n" + delegateCases + "*) echo 'unknown repository' >&2; exit 1 ;;\nesac ;;\n" + + "esac\n" + + if err := os.WriteFile(filepath.Join(bin, "rad"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + + t.Setenv("PATH", fmt.Sprintf("%s%c%s", bin, os.PathListSeparator, os.Getenv("PATH"))) + + for rid := range repos { + if err := os.MkdirAll(filepath.Join(radhome, "storage", rid), 0o755); err != nil { + t.Fatal(err) + } + } +} + +func TestScanStorage(t *testing.T) { + radhome := t.TempDir() + fakeRad(t, radhome, map[string]fakePayload{ + // same repo name upstream, distinguished purely by recorded source + "zRepoOne": {"website", "https://github.com/alice/website", true}, + "zRepoTwo": {"website", "https://gitea.local/bob/website", true}, + "zOther": {"other", "https://github.com/alice/other", true}, + // not created by gickup: no dev.gickup payload + "zForeign": {"website", "", true}, + }) + + t.Setenv("RAD_HOME", radhome) + + rid, err := scanStorage(types.Repo{Name: "website", URL: "https://gitea.local/bob/website"}, testNID, os.Environ()) + if err != nil { + t.Fatalf("scan storage: %v", err) + } + + if rid != "zRepoTwo" { + t.Errorf("expected zRepoTwo, got %s", rid) + } + + rid, err = scanStorage(types.Repo{Name: "website", URL: "https://github.com/carol/website"}, testNID, os.Environ()) + if err != nil { + t.Fatalf("scan storage: %v", err) + } + + if rid != "" { + t.Errorf("expected no match for an unmirrored upstream, got %s", rid) + } + + // credentials in the configured URL don't prevent matching the + // credential-free recorded source + rid, err = scanStorage(types.Repo{Name: "other", URL: "https://user:token@github.com/alice/other"}, testNID, os.Environ()) + if err != nil { + t.Fatalf("scan storage: %v", err) + } + + if rid != "zOther" { + t.Errorf("expected zOther, got %s", rid) + } +} + +func TestScanStorageAmbiguous(t *testing.T) { + radhome := t.TempDir() + url := "https://github.com/alice/same" + fakeRad(t, radhome, map[string]fakePayload{ + "zRepoOne": {"same", url, true}, + "zRepoTwo": {"same", url, true}, + }) + + t.Setenv("RAD_HOME", radhome) + + if _, err := scanStorage(types.Repo{Name: "same", URL: url}, testNID, os.Environ()); err == nil { + t.Error("expected an error for an ambiguous source match") + } +} + +func TestScanStorageRejectsForgedPayloads(t *testing.T) { + radhome := t.TempDir() + victim := "https://github.com/victim/foo" + fakeRad(t, radhome, map[string]fakePayload{ + // a repo seeded from the network whose crafted identity document + // claims the victim's source, but whose delegates don't include this + // profile — the delegate check must reject it + "zSeeded": {"foo", victim, false}, + }) + + t.Setenv("RAD_HOME", radhome) + + rid, err := scanStorage(types.Repo{Name: "foo", URL: victim}, testNID, os.Environ()) + if err != nil { + t.Fatalf("scan storage: %v", err) + } + + if rid != "" { + t.Errorf("forged payload was adopted: %s", rid) + } +} + +func TestMarkerStripsCredentials(t *testing.T) { + t.Parallel() + + m := marker("https://user:secret-token@github.com/alice/repo.git") + + if strings.Contains(m, "secret-token") || strings.Contains(m, "user") { + t.Errorf("marker leaks credentials: %s", m) + } + + if m != "(mirror of https://github.com/alice/repo.git)" { + t.Errorf("unexpected marker: %s", m) + } + + // non-URL remotes pass through untouched + if marker("git@github.com:alice/repo.git") != "(mirror of git@github.com:alice/repo.git)" { + t.Error("scp-style remote was altered") + } +} diff --git a/types/types.go b/types/types.go index 01a12f0b..44046ce6 100644 --- a/types/types.go +++ b/types/types.go @@ -31,6 +31,7 @@ type Destination struct { Sourcehut []GenRepo `yaml:"sourcehut"` S3 []S3Repo `yaml:"s3"` AzureBlob []AzureBlob `yaml:"azureblob"` + Radicle []Radicle `yaml:"radicle"` } // Count TODO. @@ -43,7 +44,20 @@ func (dest Destination) Count() int { len(dest.OneDev) + len(dest.Sourcehut) + len(dest.S3) + - len(dest.AzureBlob) + len(dest.AzureBlob) + + len(dest.Radicle) +} + +// Radicle : The configuration for mirroring to Radicle. +// Repositories are initialized with `rad init` and pushed into the local +// Radicle storage through the git-remote-rad helper, signed with the key of +// the profile in RAD_HOME. When that key has a passphrase, export it in the +// RAD_PASSPHRASE environment variable, which rad reads itself. +type Radicle struct { + Force bool `yaml:"force"` // overwrite diverged refs on the mirror (non-fast-forward updates) + Prune bool `yaml:"prune"` // delete refs on the mirror that no longer exist upstream + Visibility string `yaml:"visibility"` // public, private or source, default: source + Issues bool `yaml:"issues"` // [NOT YET SUPPORTED] recreate upstream issues as radicle issues (source must also fetch them with issues: true) } // Local TODO. From 2a2fd35ae35f5d86e861615bb84ed18951875f36 Mon Sep 17 00:00:00 2001 From: Thomas Christlieb Date: Thu, 30 Jul 2026 09:55:27 +0200 Subject: [PATCH 21/44] fix #414: path.Dir() does not work on windows-style paths. (#418) Co-authored-by: Thomas Christlieb --- local/local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local/local.go b/local/local.go index ca1e5819..fdea5524 100644 --- a/local/local.go +++ b/local/local.go @@ -268,7 +268,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { } if l.Keep > 0 { - parentdir := path.Dir(filepath.Join(l.Path, repo.Name)) + parentdir := filepath.Dir(filepath.Join(l.Path, repo.Name)) files, err := os.ReadDir(parentdir) if err != nil { sub.Warn(). From a7689baf636e233b6c58f691faf803a6e4857458 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:41:09 +0200 Subject: [PATCH 22/44] Bump github.com/go-git/go-git/v5 from 5.19.1 to 5.19.2 (#422) Bumps [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) from 5.19.1 to 5.19.2. - [Release notes](https://github.com/go-git/go-git/releases) - [Changelog](https://github.com/go-git/go-git/blob/main/HISTORY.md) - [Commits](https://github.com/go-git/go-git/compare/v5.19.1...v5.19.2) --- updated-dependencies: - dependency-name: github.com/go-git/go-git/v5 dependency-version: 5.19.2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index e89b9171..2cdabc1e 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/alecthomas/kong v1.15.0 github.com/bradleyfalzon/ghinstallation/v2 v2.18.0 github.com/cooperspencer/onedev v0.0.0-20240801121041-919f6d3f7ea6 - github.com/go-git/go-git/v5 v5.19.1 + github.com/go-git/go-git/v5 v5.19.2 github.com/goccy/go-yaml v1.19.2 github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 github.com/google/go-cmp v0.7.0 @@ -25,7 +25,7 @@ require ( github.com/rs/zerolog v1.35.1 github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed gitlab.com/gitlab-org/api/client-go v1.46.0 - golang.org/x/crypto v0.52.0 + golang.org/x/crypto v0.53.0 golang.org/x/oauth2 v0.36.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) @@ -87,9 +87,9 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.39.0 // indirect golang.org/x/time v0.15.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index b10fbcdb..3680bd96 100644 --- a/go.sum +++ b/go.sum @@ -72,8 +72,8 @@ github.com/go-git/go-billy/v5 v5.9.0 h1:jItGXszUDRtR/AlferWPTMN4j38BQ88XnXKbilmm github.com/go-git/go-billy/v5 v5.9.0/go.mod h1:jCnQMLj9eUgGU7+ludSTYoZL/GGmii14RxKFj7ROgHw= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= -github.com/go-git/go-git/v5 v5.19.1 h1:nX27AnaU43/K5bKktKwgBmR9lawoYVe1Ckg0rgzzN00= -github.com/go-git/go-git/v5 v5.19.1/go.mod h1:Pb1v0c7/g8aGQJwx9Us09W85yGoyvSwuhEGMH7zjDKQ= +github.com/go-git/go-git/v5 v5.19.2 h1:wkfn7vOlUBu8ivAWKBWisTiwJK4jYHzTF8Ndv1LyGqY= +github.com/go-git/go-git/v5 v5.19.2/go.mod h1:QqCBE1EFN5ddFmrliLQ3/ntRCUjZU3EJuwuB/jWEHjk= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= @@ -219,15 +219,15 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= -golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= -golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -239,16 +239,16 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= -golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From c4d33b44f9bfeb85a37321faf86b0ea591a84391 Mon Sep 17 00:00:00 2001 From: Thomas Christlieb Date: Mon, 10 Aug 2026 18:46:27 +0200 Subject: [PATCH 23/44] Fix 415 pruning old folders does count non repo folders (#420) * slightly refactor prune function for testing * add tests for pruning function * fix keep-Logic: All timestamp-prefixed-Folders will be grouped into one generation to keep. Fixes #415 Side effect: This will delete all folders `.` in the repo directory, not only `timestamp.issues` * correct tests for CI, fix typo * add test for no pruning needed --- local/local.go | 84 +++++++++++++++++++++----------------- local/local_test.go | 99 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 37 deletions(-) diff --git a/local/local.go b/local/local.go index fdea5524..409780f0 100644 --- a/local/local.go +++ b/local/local.go @@ -268,57 +268,67 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { } if l.Keep > 0 { - parentdir := filepath.Dir(filepath.Join(l.Path, repo.Name)) - files, err := os.ReadDir(parentdir) - if err != nil { + if err := pruneOldBackups(filepath.Dir(filepath.Join(l.Path, repo.Name)), repo.Name, l); err != nil { sub.Warn(). Str("repo", repo.Name).Msg(err.Error()) return false } + } - keep := []string{} - for _, file := range files { - fname := file.Name() - if l.Zip { - fname = strings.TrimSuffix(file.Name(), ".zip") - } + x = 5 + } - fname = strings.TrimSuffix(fname, ".issues") + return true +} - _, err := strconv.ParseInt(fname, 10, 64) - if err != nil { - sub.Warn(). - Str("repo", repo.Name). - Msgf("couldn't parse timestamp! %s", types.Red(file.Name())) - continue - } - if l.Zip && !strings.HasSuffix(file.Name(), ".zip") { - continue - } - keep = append(keep, file.Name()) - } +// pruneOldBackups enforces l.Keep by removing the oldest timestamped backups +func pruneOldBackups(parentdir string, repoName string, l types.Local) error { + files, err := os.ReadDir(parentdir) + if err != nil { + return err + } - sort.Sort(sort.Reverse(sort.StringSlice(keep))) + // A backup generation is the "" directory plus "." + // (like ".issues") + // They are grouped by their timestamp so that together they only count as one backup against l.Keep. + generations := map[int64][]string{} + for _, file := range files { + stamp, _, _ := strings.Cut(file.Name(), ".") - if len(keep) > l.Keep { - toremove := keep[l.Keep:] - for _, file := range toremove { - sub.Info(). - Msgf("removing %s", types.Red(path.Join(parentdir, file))) - err := os.RemoveAll(path.Join(parentdir, file)) - if err != nil { - sub.Warn(). - Str("repo", repo.Name).Msg(err.Error()) - } - } - } + ts, err := strconv.ParseInt(stamp, 10, 64) + if err != nil { + sub.Warn(). + Str("repo", repoName). + Msgf("couldn't parse timestamp! %s", types.Red(file.Name())) + continue } + generations[ts] = append(generations[ts], file.Name()) + } - x = 5 + if len(generations) <= l.Keep { + return nil } - return true + stamps := make([]int64, 0, len(generations)) + for ts := range generations { + stamps = append(stamps, ts) + } + sort.Slice(stamps, func(i, j int) bool { return stamps[i] > stamps[j] }) + + for _, ts := range stamps[l.Keep:] { + for _, file := range generations[ts] { + sub.Info(). + Msgf("removing %s", types.Red(filepath.Join(parentdir, file))) + err := os.RemoveAll(filepath.Join(parentdir, file)) + if err != nil { + sub.Warn(). + Str("repo", repoName).Msg(err.Error()) + } + } + } + + return nil } func updateRepository(reponame string, auth transport.AuthMethod, dry bool, l types.Local) error { diff --git a/local/local_test.go b/local/local_test.go index fa1d730f..bfe2a067 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -1,12 +1,111 @@ package local import ( + "os" + "path/filepath" + "reflect" "strings" "testing" "github.com/cooperspencer/gickup/types" ) +func runPrune(t *testing.T, backups []string, keep int) []string { + t.Helper() + + parentdir := t.TempDir() + for _, d := range backups { + if err := os.Mkdir(filepath.Join(parentdir, d), 0o777); err != nil { + t.Fatal(err) + } + } + + if err := pruneOldBackups(parentdir, "test/repo", types.Local{Keep: keep}); err != nil { + t.Fatal(err) + } + + files, err := os.ReadDir(parentdir) + if err != nil { + t.Fatal(err) + } + remaining := []string{} + for _, f := range files { + remaining = append(remaining, f.Name()) + } + + return remaining +} + +func TestPruneKeepsNewestBackup(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, []string{"1785061240", "1785115104"}, 1) + + want := []string{"1785115104"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + +func TestPruneKeepsIssuesDirWithItsBackup(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, + []string{"1785061241", "1785061241.issues", "1785115111", "1785115111.issues"}, 1) + + want := []string{"1785115111", "1785115111.issues"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + +func TestPruneKeepsOtherDirectories(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, + []string{"1785061241", "1785061241.issues", "dummy", "dummy.issue", "1785061241x.dummy", "1785115111", "1785115111.issues"}, 1) + + want := []string{"1785061241x.dummy", "1785115111", "1785115111.issues", "dummy", "dummy.issue"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + +func TestPruneDeletesOtherTimestampedDirectories(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, + []string{"1785061241", "1785061241.issues", "1785061241.dummy", "1785115111", "1785115111.issues"}, 1) + + want := []string{"1785115111", "1785115111.issues"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + +func TestPruneKeepsMultipleGenerations(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, + []string{"1785000000", "1785000000.issues", "1785061241", "1785061241.issues", "1785115111", "1785115111.issues"}, 2) + + want := []string{"1785061241", "1785061241.issues", "1785115111", "1785115111.issues"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + +func TestPruneKeepsEverythingBelowKeepCount(t *testing.T) { + t.Parallel() + + remaining := runPrune(t, []string{"1785115111", "1785115111.issues"}, 2) + + want := []string{"1785115111", "1785115111.issues"} + if !reflect.DeepEqual(remaining, want) { + t.Fatalf("remaining = %v, want %v", remaining, want) + } +} + func TestRandomStringLengthAndCharset(t *testing.T) { t.Parallel() From 9fc59f0c48336a55bed1f5eb8bff7c28822538a5 Mon Sep 17 00:00:00 2001 From: gdm257 Date: Wed, 19 Aug 2026 01:54:39 +0800 Subject: [PATCH 24/44] Support WebDAV destination (#359) * feat(destination): add WebDAV destination Add a stdlib-only WebDAV destination mirroring the s3/azureblob flow: bare-clone into a temp dir, optional zip, upload via HTTP PUT with Basic auth, and prune stale remote files via PROPFIND/DELETE. - types: WebDAVRepo struct, Destination.WebDAV field, Count() - webdav: new package (UploadDirToWebDAV, DeleteObjectsNotInRepo) - main: dispatch loop on par with azureblob - conf.example.yml + gickup_spec.json: document the new destination * feat: auto retry Auto retry network errors and some situations. Servers that disable Depth: infinity (Sabre/DAV, Nextcloud and others answer 400/403/501) failed listing and could not prune stale files. Try infinity once, fall back to recursive PROPFIND walk on rejection. * refactor: use gowebdav client Classify retry errors and make backoff interruptible. --- conf.example.yml | 8 ++ gickup_spec.json | 38 ++++++++ go.mod | 1 + go.sum | 2 + main.go | 86 ++++++++++++++++++ types/types.go | 32 +++++-- webdav/webdav.go | 207 ++++++++++++++++++++++++++++++++++++++++++ webdav/webdav_test.go | 88 ++++++++++++++++++ 8 files changed, 452 insertions(+), 10 deletions(-) create mode 100644 webdav/webdav.go create mode 100644 webdav/webdav_test.go diff --git a/conf.example.yml b/conf.example.yml index ca3cfc19..028e3663 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -326,6 +326,14 @@ destination: structured: true # checks repos out like hostersite/user|organization/repo zip: false # if true, will zip the entire git repo into a single zip file and upload that instead datecreatedir: false # if true, gickup will create a directory for backup with the creation date + webdav: + - url: https://webdav.example.com/dav # the WebDAV endpoint to upload the backups to (e.g. Nextcloud, Apache mod_dav, rclone serve webdav) + username: your-username # HTTP Basic auth username + password: your-password # HTTP Basic auth password + path: repos # subdirectory on the WebDAV server to store the backups under (repos end up under url/path/) + structured: true # checks repos out like hostersite/user|organization/repo + zip: false # if true, will zip the entire git repo into a single zip file and upload that instead + datecreatedir: false # if true, gickup will create a directory for backup with the creation date cron: 0 22 * * * # optional - when cron is not provided, the program runs once and exits. # Otherwise, it runs according to the cron schedule. # See timezone commentary in docker-compose.yml for making sure this container runs diff --git a/gickup_spec.json b/gickup_spec.json index 3ba70423..1200bd10 100644 --- a/gickup_spec.json +++ b/gickup_spec.json @@ -860,6 +860,44 @@ "additionalProperties": false } }, + "webdav": { + "type": "array", + "description": "Upload bare-cloned repositories to a WebDAV server using HTTP Basic auth. Stale remote files are pruned via PROPFIND/DELETE.", + "items": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The WebDAV endpoint URL to upload the backups to (e.g. a Nextcloud, Apache mod_dav or rclone serve webdav server)" + }, + "username": { + "type": "string", + "description": "HTTP Basic auth username" + }, + "password": { + "type": "string", + "description": "HTTP Basic auth password" + }, + "path": { + "type": "string", + "description": "Subdirectory on the WebDAV server to store the backups under; repositories end up under url/path/" + }, + "structured": { + "type": "boolean", + "description": "if set to `true`, it checks out the repos in a more structured way, like `hoster/user|organization/repository`" + }, + "zip": { + "type": "boolean", + "description": "If true, zip the entire Git repo into a single zip file and upload that instead" + }, + "datecreatedir": { + "type": "boolean", + "description": "If true, gickup will create a directory for backup with the creation date" + } + }, + "additionalProperties": false + } + }, "radicle": { "type": "array", "description": "Mirror repositories into the storage of a local Radicle node. Requires the `git`, `rad` and `git-remote-rad` binaries.", diff --git a/go.mod b/go.mod index 2cdabc1e..7f7658df 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/robfig/cron/v3 v3.0.1 github.com/rs/zerolog v1.35.1 github.com/shurcooL/githubv4 v0.0.0-20260209031235-2402fdf4a9ed + github.com/studio-b12/gowebdav v0.13.0 gitlab.com/gitlab-org/api/client-go v1.46.0 golang.org/x/crypto v0.53.0 golang.org/x/oauth2 v0.36.0 diff --git a/go.sum b/go.sum index 3680bd96..b6e1f86c 100644 --- a/go.sum +++ b/go.sum @@ -201,6 +201,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/studio-b12/gowebdav v0.13.0 h1:OcwSg6IQHOFNdYHn3bPOHwSE8looG8N56Y5xTT1asqQ= +github.com/studio-b12/gowebdav v0.13.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= diff --git a/main.go b/main.go index 09be5997..c64b3a71 100644 --- a/main.go +++ b/main.go @@ -32,6 +32,7 @@ import ( "github.com/cooperspencer/gickup/s3" "github.com/cooperspencer/gickup/sourcehut" "github.com/cooperspencer/gickup/types" + "github.com/cooperspencer/gickup/webdav" "github.com/cooperspencer/gickup/whatever" "github.com/cooperspencer/gickup/zip" "github.com/go-git/go-git/v5" @@ -424,6 +425,91 @@ func backup(repos []types.Repo, conf *types.Conf) { } } + for _, d := range conf.Destination.WebDAV { + repotime := time.Now() + status := 0 + + logOp := "uploading" + if d.Zip { + logOp = "zipping and uploading" + } + log.Info(). + Str("stage", "webdav"). + Str("url", d.Url). + Msgf("%s %s to %s", logOp, types.Blue(r.Name), d.Url) + + if !cli.Dry { + tempname := fmt.Sprintf("webdav-%x", repotime) + tempdir, err := os.MkdirTemp(os.TempDir(), tempname) + if err != nil { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Msg(err.Error()) + continue + } + + if d.Structured { + r.Name = path.Join(r.Hoster, r.Owner, r.Name) + } + + if d.DateCreateDir { + r.Name = currentDateDir + r.Name + } + + defer os.RemoveAll(tempdir) + tempClonePath := path.Join(tempdir, r.Name) + _, err = local.TempCloneBare(r, tempClonePath) + if err != nil { + switch { + case errors.Is(err, git.NoErrAlreadyUpToDate): + log.Info(). + Str("stage", "webdav"). + Msg(err.Error()) + case errors.Is(err, transport.ErrEmptyRemoteRepository): + log.Warn(). + Str("repo", r.Name). + Msgf("%s - Skipping backup", err.Error()) + continue + default: + log.Error(). + Str("stage", "tempclone"). + Str("git", "clone"). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + } + + if d.Zip { + log.Info(). + Msgf("zipping %s", types.Green(r.Name)) + err := zip.Zip(tempClonePath, []string{tempClonePath}) + if err != nil { + log.Error(). + Str("stage", "zip"). + Str("repo", r.Name). + Msg(err.Error()) + log.Error().Msgf("Skipping backup of %s due to error while zipping", r.Name) + continue + } + } + err = webdav.UploadDirToWebDAV(tempdir, d) + if err != nil { + log.Error().Str("stage", "webdav").Str("url", d.Url).Msg(err.Error()) + } + err = webdav.DeleteObjectsNotInRepo(tempdir, r.Name, d) + if err != nil { + log.Error().Str("stage", "webdav").Str("url", d.Url).Msg(err.Error()) + } + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "webdav", d.Url).Set(time.Since(repotime).Seconds()) + status = 1 + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "webdav", d.Url).Set(float64(status)) + prometheus.DestinationBackupsComplete.WithLabelValues("webdav").Inc() + } + } + for _, d := range conf.Destination.Gitea { if d.MirrorInterval != "" { log.Warn(). diff --git a/types/types.go b/types/types.go index 44046ce6..8aba30df 100644 --- a/types/types.go +++ b/types/types.go @@ -22,16 +22,17 @@ import ( // Destination TODO. type Destination struct { - Gitlab []GenRepo `yaml:"gitlab"` - Local []Local `yaml:"local"` - Github []GenRepo `yaml:"github"` - Gitea []GenRepo `yaml:"gitea"` - Gogs []GenRepo `yaml:"gogs"` - OneDev []GenRepo `yaml:"onedev"` - Sourcehut []GenRepo `yaml:"sourcehut"` - S3 []S3Repo `yaml:"s3"` - AzureBlob []AzureBlob `yaml:"azureblob"` - Radicle []Radicle `yaml:"radicle"` + Gitlab []GenRepo `yaml:"gitlab"` + Local []Local `yaml:"local"` + Github []GenRepo `yaml:"github"` + Gitea []GenRepo `yaml:"gitea"` + Gogs []GenRepo `yaml:"gogs"` + OneDev []GenRepo `yaml:"onedev"` + Sourcehut []GenRepo `yaml:"sourcehut"` + S3 []S3Repo `yaml:"s3"` + AzureBlob []AzureBlob `yaml:"azureblob"` + WebDAV []WebDAVRepo `yaml:"webdav"` + Radicle []Radicle `yaml:"radicle"` } // Count TODO. @@ -45,6 +46,7 @@ func (dest Destination) Count() int { len(dest.Sourcehut) + len(dest.S3) + len(dest.AzureBlob) + + len(dest.WebDAV) + len(dest.Radicle) } @@ -614,3 +616,13 @@ type AzureBlob struct { Zip bool `yaml:"zip"` DateCreateDir bool `yaml:"datecreatedir"` } + +type WebDAVRepo struct { + Url string `yaml:"url"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Path string `yaml:"path"` + Structured bool `yaml:"structured"` + Zip bool `yaml:"zip"` + DateCreateDir bool `yaml:"datecreatedir"` +} diff --git a/webdav/webdav.go b/webdav/webdav.go new file mode 100644 index 00000000..fe7c1895 --- /dev/null +++ b/webdav/webdav.go @@ -0,0 +1,207 @@ +package webdav + +import ( + "context" + "errors" + "fmt" + "io" + "math/rand/v2" + "net" + "net/http" + "os" + "path" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/cooperspencer/gickup/logger" + "github.com/cooperspencer/gickup/types" + "github.com/rs/zerolog" + gowebdav "github.com/studio-b12/gowebdav" +) + +var sub zerolog.Logger + +func newClient(repo types.WebDAVRepo) *gowebdav.Client { + t, ok := http.DefaultTransport.(*http.Transport) + if !ok { + t = &http.Transport{} + } + t = t.Clone() + t.ResponseHeaderTimeout = 60 * time.Second + c := gowebdav.NewClient(repo.Url, repo.Username, repo.Password) + c.SetTransport(&retryRoundTripper{base: t}) + return c +} + +// UploadDirToWebDAV uploads the contents of a directory to a WebDAV server +func UploadDirToWebDAV(directory string, repo types.WebDAVRepo) error { + sub = logger.CreateSubLogger("stage", "webdav", "url", repo.Url) + c := newClient(repo) + + return filepath.Walk(directory, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || info.Mode()&os.ModeSymlink != 0 { + return nil + } + rel := filepath.ToSlash(p[len(directory)+1:]) + remote := path.Join(repo.Path, rel) + + f, err := os.Open(p) + if err != nil { + return err + } + defer f.Close() + return c.WriteStreamWithLength(remote, f, info.Size(), 0o644) + }) +} + +// DeleteObjectsNotInRepo deletes files from the WebDAV server that are not present in the repository +func DeleteObjectsNotInRepo(directory, repoName string, repo types.WebDAVRepo) error { + sub = logger.CreateSubLogger("stage", "webdav", "url", repo.Url) + c := newClient(repo) + root := path.Join(repo.Path, repoName) + + keys, err := walkRemote(c, root) + if err != nil { + // Nothing uploaded yet. + if gowebdav.IsErrNotFound(err) { + return nil + } + return err + } + for _, key := range keys { + if _, statErr := os.Stat(filepath.Join(directory, repoName, filepath.FromSlash(key))); statErr == nil { + continue + } else if !os.IsNotExist(statErr) { + return statErr + } + sub.Debug().Msgf("Removing %s from webdav", key) + if err := c.RemoveAll(path.Join(root, key)); err != nil { + return err + } + } + + return nil +} + +// walkRemote returns the keys relative to dir of every regular file beneath it. +func walkRemote(c *gowebdav.Client, dir string) ([]string, error) { + var files []string + prefix := strings.TrimSuffix(dir, "/") + "/" + visited := map[string]bool{} + + var walk func(current string) error + walk = func(current string) error { + if visited[current] { + return nil + } + visited[current] = true + entries, err := c.ReadDir(current) + if err != nil { + return err + } + for _, e := range entries { + child := path.Join(current, e.Name()) + if e.IsDir() { + if err := walk(child); err != nil { + return err + } + continue + } + files = append(files, strings.TrimPrefix(child, prefix)) + } + return nil + } + if err := walk(dir); err != nil { + return nil, err + } + return files, nil +} + +// retryRoundTripper retries transient failures with exponential backoff, +// rewinding seekable bodies via GetBody so PUTs replay the full payload. +type retryRoundTripper struct { + base http.RoundTripper +} + +func (r *retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + const maxAttempts = 3 + backoff := 500 * time.Millisecond + + resp, err := r.base.RoundTrip(req) + for attempt := 1; shouldRetry(resp, err) && attempt < maxAttempts; attempt++ { + // Drain and close the failed response so its connection can be reused. + if resp != nil { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 512)) + resp.Body.Close() + } + + // Rewind seekable bodies via GetBody so retried PUTs replay the full payload. + if req.GetBody != nil { + body, berr := req.GetBody() + if berr != nil { + return nil, fmt.Errorf("retry %s %s: rewind body: %w", req.Method, req.URL.Redacted(), berr) + } + req.Body = body + } + + wait := backoff + time.Duration(rand.Int64N(int64(backoff/2))) + reason := "unknown" + switch { + case err != nil: + reason = err.Error() + case resp != nil: + reason = resp.Status + } + sub.Warn().Msgf("%s %s: attempt %d failed (%s); retrying in %v", + req.Method, req.URL.Redacted(), attempt, reason, wait) + + // Back off with jitter, bailing out immediately if the request was cancelled. + timer := time.NewTimer(wait) + select { + case <-timer.C: + case <-req.Context().Done(): + timer.Stop() + return resp, req.Context().Err() + } + backoff *= 2 + + resp, err = r.base.RoundTrip(req) + } + return resp, err +} + +func shouldRetry(resp *http.Response, err error) bool { + if err != nil { + // Context cancellations are intentional, not transient + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return false + } + // Only retry transient network failures + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return true + } + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return true + } + if errors.Is(err, syscall.ECONNRESET) || errors.Is(err, syscall.ECONNREFUSED) || + errors.Is(err, syscall.EPIPE) { + return true + } + return false + } + switch resp.StatusCode { + case http.StatusTooManyRequests, // 429 + http.StatusInternalServerError, // 500 + http.StatusBadGateway, // 502 + http.StatusServiceUnavailable, // 503 + http.StatusGatewayTimeout: // 504 + return true + } + return false +} diff --git a/webdav/webdav_test.go b/webdav/webdav_test.go new file mode 100644 index 00000000..f93d41b0 --- /dev/null +++ b/webdav/webdav_test.go @@ -0,0 +1,88 @@ +package webdav + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "syscall" + "testing" + "time" +) + +// fakeNetError is a minimal net.Error for table tests. +type fakeNetError struct{ timeout bool } + +func (fakeNetError) Error() string { return "net error" } +func (e fakeNetError) Timeout() bool { return e.timeout } +func (fakeNetError) Temporary() bool { return false } + +func TestShouldRetry(t *testing.T) { + t.Parallel() + cases := []struct { + name string + resp *http.Response + err error + want bool + }{ + {"503", &http.Response{StatusCode: http.StatusServiceUnavailable}, nil, true}, + {"429", &http.Response{StatusCode: http.StatusTooManyRequests}, nil, true}, + {"500", &http.Response{StatusCode: http.StatusInternalServerError}, nil, true}, + {"502", &http.Response{StatusCode: http.StatusBadGateway}, nil, true}, + {"504", &http.Response{StatusCode: http.StatusGatewayTimeout}, nil, true}, + {"200", &http.Response{StatusCode: http.StatusOK}, nil, false}, + {"404", &http.Response{StatusCode: http.StatusNotFound}, nil, false}, + {"401", &http.Response{StatusCode: http.StatusUnauthorized}, nil, false}, + {"net timeout", nil, fakeNetError{timeout: true}, true}, + {"eof", nil, io.EOF, true}, + {"unexpected eof", nil, io.ErrUnexpectedEOF, true}, + {"conn reset", nil, syscall.ECONNRESET, true}, + {"conn refused", nil, syscall.ECONNREFUSED, true}, + {"broken pipe", nil, syscall.EPIPE, true}, + {"wrapped eof", nil, fmt.Errorf("wrap: %w", io.EOF), true}, + {"context canceled", nil, context.Canceled, false}, + {"context deadline", nil, context.DeadlineExceeded, false}, + {"plain error", nil, errors.New("boom"), false}, + {"non-timeout net", nil, fakeNetError{timeout: false}, false}, + } + for _, c := range cases { + if got := shouldRetry(c.resp, c.err); got != c.want { + t.Errorf("%s: got %v want %v", c.name, got, c.want) + } + } +} + +// TestRetryAbortsOnContextCancel verifies the backoff sleep yields to a +// canceled request context instead of blocking for the full retry window. +func TestRetryAbortsOnContextCancel(t *testing.T) { + t.Parallel() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + time.AfterFunc(60*time.Millisecond, cancel) + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL, nil) + if err != nil { + t.Fatal(err) + } + + start := time.Now() + resp, err := (&retryRoundTripper{base: http.DefaultTransport}).RoundTrip(req) + elapsed := time.Since(start) + if resp != nil { + resp.Body.Close() + } + + if err == nil { + t.Fatal("expected error from canceled context") + } + if elapsed > 400*time.Millisecond { + t.Fatalf("retry did not abort on cancel: %v (backoff base is 500ms)", elapsed) + } +} From ed581d75d93fc018f32cc4ba7a3bcd2bf68613b1 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 19 Aug 2026 02:34:43 +0800 Subject: [PATCH 25/44] fix GitLab mirroring visibility, URL parsing, and ref pushes (#423) --- gitlab/gitlab.go | 28 ++++++++++------- gitlab/gitlab_test.go | 31 ++++++++++++++++++ local/local.go | 36 +++++++++++++++++++-- local/local_test.go | 46 +++++++++++++++++++++++++++ types/types.go | 20 +++++++----- types/types_test.go | 9 ++++-- whatever/whatever.go | 39 ++++++++++++++--------- whatever/whatever_test.go | 66 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 236 insertions(+), 39 deletions(-) create mode 100644 gitlab/gitlab_test.go diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index ff78c514..c32d8b1f 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -16,6 +16,20 @@ import ( var sub zerolog.Logger +func getRepoVisibility(visibility string, private bool) gitlab.VisibilityValue { + switch strings.ToLower(strings.TrimSpace(visibility)) { + case "private": + return gitlab.PrivateVisibility + case "public": + return gitlab.PublicVisibility + default: + if private { + return gitlab.PrivateVisibility + } + return gitlab.PublicVisibility + } +} + // Backup TODO. func Backup(r types.Repo, d types.GenRepo, dry bool) bool { var gitlabclient *gitlab.Client @@ -69,13 +83,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { splittedurl[0], r.Owner, r.Token, splittedurl[1]) } - var visibility gitlab.VisibilityValue - - if r.Private { - visibility = gitlab.PrivateVisibility - } else { - visibility = gitlab.PublicVisibility - } + visibility := getRepoVisibility(d.Visibility.Repositories, r.Private) opts := &gitlab.CreateProjectOptions{ Mirror: &True, @@ -586,11 +594,7 @@ func GetIssues(repo *gitlab.Project, client *gitlab.Client, conf types.GenRepo) // GetOrCreate Get or create a repository func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { - visibility := gitlab.PublicVisibility - - if repo.Private { - visibility = gitlab.PrivateVisibility - } + visibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private) sub = logger.CreateSubLogger("stage", "gitlab", "url", destination.URL) diff --git a/gitlab/gitlab_test.go b/gitlab/gitlab_test.go new file mode 100644 index 00000000..c05d5a95 --- /dev/null +++ b/gitlab/gitlab_test.go @@ -0,0 +1,31 @@ +package gitlab + +import ( + "testing" + + gitlab "gitlab.com/gitlab-org/api/client-go" +) + +func TestGetRepoVisibility(t *testing.T) { + tests := []struct { + name string + configured string + sourcePrivate bool + want gitlab.VisibilityValue + }{ + {name: "private overrides public source", configured: "private", want: gitlab.PrivateVisibility}, + {name: "public overrides private source", configured: "public", sourcePrivate: true, want: gitlab.PublicVisibility}, + {name: "empty follows private source", sourcePrivate: true, want: gitlab.PrivateVisibility}, + {name: "empty follows public source", want: gitlab.PublicVisibility}, + {name: "source follows private source", configured: "source", sourcePrivate: true, want: gitlab.PrivateVisibility}, + {name: "source follows public source", configured: "source", want: gitlab.PublicVisibility}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := getRepoVisibility(test.configured, test.sourcePrivate); got != test.want { + t.Fatalf("getRepoVisibility(%q, %t) = %q, want %q", test.configured, test.sourcePrivate, got, test.want) + } + }) + } +} diff --git a/local/local.go b/local/local.go index 409780f0..27d1236a 100644 --- a/local/local.go +++ b/local/local.go @@ -724,13 +724,45 @@ func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url strin err = repo.Push(&pushoptions) if err == nil || errors.Is(err, git.NoErrAlreadyUpToDate) { - pushoptions = git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}} + refspecs, err := getPushRefSpecs(repo, headref.Name()) + if err != nil { + return err + } - return repo.Push(&pushoptions) + var pushErrors []error + // Keep valid refs moving when a destination hook rejects one branch or tag. + for _, refspec := range refspecs { + pushoptions.RefSpecs = []config.RefSpec{refspec} + err = repo.Push(&pushoptions) + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + pushErrors = append(pushErrors, err) + } + } + + return errors.Join(pushErrors...) } return err } +func getPushRefSpecs(repo *git.Repository, skip plumbing.ReferenceName) ([]config.RefSpec, error) { + refs, err := repo.References() + if err != nil { + return nil, err + } + + refspecs := []config.RefSpec{} + err = refs.ForEach(func(ref *plumbing.Reference) error { + name := ref.Name() + if name == skip || (!name.IsBranch() && !name.IsTag()) { + return nil + } + + refspecs = append(refspecs, config.RefSpec(fmt.Sprintf("%s:%s", name, name))) + return nil + }) + return refspecs, err +} + func RandomString(length int) string { charset := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" seededRand := rand.New(rand.NewSource(time.Now().UnixNano())) diff --git a/local/local_test.go b/local/local_test.go index bfe2a067..fb10515a 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -8,6 +8,9 @@ import ( "testing" "github.com/cooperspencer/gickup/types" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/storage/memory" ) func runPrune(t *testing.T, backups []string, keep int) []string { @@ -180,3 +183,46 @@ func TestTokenAuth_WithTokenUser(t *testing.T) { t.Errorf("Password = %q, want %q", got.Password, "mytoken") } } + +func TestGetPushRefSpecs(t *testing.T) { + t.Parallel() + + repo, err := git.Init(memory.NewStorage(), nil) + if err != nil { + t.Fatalf("init repository: %v", err) + } + + hash := plumbing.NewHash("1111111111111111111111111111111111111111") + for _, name := range []plumbing.ReferenceName{ + plumbing.NewBranchReferenceName("master"), + plumbing.NewBranchReferenceName("feature"), + plumbing.NewTagReferenceName("v1.0.0"), + plumbing.NewRemoteReferenceName("origin", "feature"), + } { + if err := repo.Storer.SetReference(plumbing.NewHashReference(name, hash)); err != nil { + t.Fatalf("set reference %s: %v", name, err) + } + } + + refspecs, err := getPushRefSpecs(repo, plumbing.NewBranchReferenceName("master")) + if err != nil { + t.Fatalf("getPushRefSpecs() error: %v", err) + } + + want := map[string]bool{ + "refs/heads/feature:refs/heads/feature": false, + "refs/tags/v1.0.0:refs/tags/v1.0.0": false, + } + for _, refspec := range refspecs { + key := refspec.String() + if _, ok := want[key]; !ok { + t.Fatalf("unexpected refspec %q", key) + } + want[key] = true + } + for refspec, found := range want { + if !found { + t.Errorf("missing refspec %q", refspec) + } + } +} diff --git a/types/types.go b/types/types.go index 8aba30df..b9c6824b 100644 --- a/types/types.go +++ b/types/types.go @@ -454,20 +454,24 @@ type GithubDestination struct { } // GetHost TODO. -func GetHost(url string) string { - if strings.Contains(url, "http://") { - url = strings.Split(url, "http://")[1] +func GetHost(rawURL string) string { + if endpoint, err := transport.NewEndpoint(rawURL); err == nil && endpoint.Host != "" { + return endpoint.Host } - if strings.Contains(url, "https://") { - url = strings.Split(url, "https://")[1] + if strings.Contains(rawURL, "http://") { + rawURL = strings.Split(rawURL, "http://")[1] } - if strings.Contains(url, "/") { - url = strings.Split(url, "/")[0] + if strings.Contains(rawURL, "https://") { + rawURL = strings.Split(rawURL, "https://")[1] } - return url + if strings.Contains(rawURL, "/") { + rawURL = strings.Split(rawURL, "/")[0] + } + + return rawURL } // GetValues TODO. diff --git a/types/types_test.go b/types/types_test.go index 44008392..6195145a 100644 --- a/types/types_test.go +++ b/types/types_test.go @@ -223,9 +223,12 @@ func TestGetHost(t *testing.T) { t.Parallel() tests := map[string]string{ - "https://example.com/org/repo": "example.com", - "http://example.com/org/repo": "example.com", - "example.com/org/repo": "example.com", + "https://example.com/org/repo": "example.com", + "http://example.com/org/repo": "example.com", + "ssh://alice%40example.com@git.example.com/org/repo.git": "git.example.com", + "ssh://git@example.com:2222/org/repo.git": "example.com", + "git@example.com:org/repo.git": "example.com", + "example.com/org/repo": "example.com", } for input, want := range tests { diff --git a/whatever/whatever.go b/whatever/whatever.go index df926828..fbfb12f8 100644 --- a/whatever/whatever.go +++ b/whatever/whatever.go @@ -1,13 +1,35 @@ package whatever import ( - "os" + "path" + "path/filepath" "strings" "github.com/cooperspencer/gickup/types" + "github.com/go-git/go-git/v5/plumbing/transport" "github.com/rs/zerolog/log" ) +func getRepoNameAndHost(rawURL string) (string, string) { + endpoint, err := transport.NewEndpoint(rawURL) + if err == nil { + if endpoint.Protocol == "file" { + name := filepath.Base(filepath.Clean(endpoint.Path)) + return strings.TrimSuffix(name, ".git"), "local" + } + + repoPath := endpoint.Path + if index := strings.IndexAny(repoPath, "?#"); index >= 0 { + repoPath = repoPath[:index] + } + name := path.Base(strings.TrimRight(repoPath, "/")) + return strings.TrimSuffix(name, ".git"), endpoint.Host + } + + name := filepath.Base(filepath.Clean(rawURL)) + return strings.TrimSuffix(name, ".git"), "local" +} + // Get TODO. func Get(conf *types.Conf) ([]types.Repo, bool) { ran := false @@ -22,9 +44,9 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { log.Error(). Str("stage", "whatever"). Msg("no url configured") + continue } - hoster := "local" if repo.User == "" { if repo.Username != "" { repo.User = repo.Username @@ -32,18 +54,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { repo.User = "git" } } - if _, err := os.Stat(repo.URL); os.IsNotExist(err) { - hoster = types.GetHost(repo.URL) - } - - separator := "/" - if hoster == "local" { - separator = string(os.PathSeparator) - } - name := repo.URL[strings.LastIndex(repo.URL, separator)+1:] - if strings.HasSuffix(name, ".git") { - name = name[:strings.LastIndex(name, ".git")] - } + name, hoster := getRepoNameAndHost(repo.URL) repos = append(repos, types.Repo{ Name: name, diff --git a/whatever/whatever_test.go b/whatever/whatever_test.go index 98ae589a..d5998ad7 100644 --- a/whatever/whatever_test.go +++ b/whatever/whatever_test.go @@ -32,6 +32,54 @@ func TestGetBuildsRemoteRepo(t *testing.T) { } } +func TestGetBuildsSSHRemoteRepo(t *testing.T) { + t.Parallel() + + conf := &types.Conf{ + Source: types.Source{ + Any: []types.GenRepo{{URL: "ssh://alice%40example.com@git.example.com/org/repo.git"}}, + }, + } + + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + + if len(repos) != 1 { + t.Fatalf("expected 1 repo, got %d", len(repos)) + } + + repo := repos[0] + if repo.Name != "repo" || repo.Hoster != "git.example.com" || repo.Owner != "git" { + t.Fatalf("unexpected repo: %#v", repo) + } +} + +func TestGetBuildsSCPRemoteRepo(t *testing.T) { + t.Parallel() + + conf := &types.Conf{ + Source: types.Source{ + Any: []types.GenRepo{{URL: "git@git.example.com:org/repo.git"}}, + }, + } + + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + + if len(repos) != 1 { + t.Fatalf("expected 1 repo, got %d", len(repos)) + } + + repo := repos[0] + if repo.Name != "repo" || repo.Hoster != "git.example.com" || repo.Owner != "git" { + t.Fatalf("unexpected repo: %#v", repo) + } +} + func TestGetBuildsLocalRepo(t *testing.T) { t.Parallel() @@ -60,3 +108,21 @@ func TestGetBuildsLocalRepo(t *testing.T) { t.Fatalf("unexpected repo: %#v", repo) } } + +func TestGetSkipsMissingURL(t *testing.T) { + t.Parallel() + + conf := &types.Conf{ + Source: types.Source{ + Any: []types.GenRepo{{}}, + }, + } + + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + if len(repos) != 0 { + t.Fatalf("expected no repositories, got %d", len(repos)) + } +} From 7d810a4a1b3fc2b828d291a466b53185338c772f Mon Sep 17 00:00:00 2001 From: Dmitriy Prikhodko Date: Wed, 19 Aug 2026 12:46:36 +0500 Subject: [PATCH 26/44] Fix GitHub App authentication for LFS-backed local mirrors (#425) * fix: use fresh GitHub App auth for LFS mirror updates * fix: satisfy lint checks --- gitcmd/gitcmd.go | 74 ++++++++--- gitcmd/gitcmd_test.go | 284 ++++++++++++++++++++++++++++++++++++++++++ local/local.go | 108 +++++++++------- local/local_test.go | 188 ++++++++++++++++++++++++++++ 4 files changed, 588 insertions(+), 66 deletions(-) create mode 100644 gitcmd/gitcmd_test.go diff --git a/gitcmd/gitcmd.go b/gitcmd/gitcmd.go index 3169c2e4..f8b3aec6 100644 --- a/gitcmd/gitcmd.go +++ b/gitcmd/gitcmd.go @@ -2,6 +2,7 @@ package gitcmd import ( "context" + "encoding/base64" "errors" "fmt" "os" @@ -9,6 +10,23 @@ import ( "strings" ) +type Auth struct { + Username string + Password string +} + +func (a *Auth) Env() []string { + if a == nil || (a.Username == "" && a.Password == "") { + return nil + } + encoded := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.Username, a.Password))) + return []string{ + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=http.extraHeader", + fmt.Sprintf("GIT_CONFIG_VALUE_0=Authorization: Basic %s", encoded), + } +} + type GitCmd struct { CMD string } @@ -23,55 +41,64 @@ func New() (GitCmd, error) { return GitCmd{CMD: "git"}, nil } -func (g GitCmd) Clone(url, reponame string, bare bool, mirror bool) error { - cmd := exec.CommandContext(context.Background(), g.CMD, "clone", url, reponame) +func (g GitCmd) Command(ctx context.Context, auth *Auth, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, g.CMD, args...) + if authEnv := auth.Env(); len(authEnv) > 0 { + cmd.Env = append(os.Environ(), authEnv...) + } + return cmd +} + +func (g GitCmd) Clone(url, reponame string, bare bool, mirror bool, auth *Auth) error { + args := []string{"clone", url, reponame} if bare { - cmd.Args = append(cmd.Args, "--bare") + args = append(args, "--bare") } if mirror { - cmd.Args = append(cmd.Args, "--mirror") + args = append(args, "--mirror") } + cmd := g.Command(context.Background(), auth, args...) return cmd.Run() } -func (g GitCmd) Pull(bare bool, mirror bool, repopath string) error { +func (g GitCmd) Pull(bare bool, mirror bool, repopath string, auth *Auth) error { var args []string if bare || mirror { args = []string{"-C", repopath, "fetch", "--all"} } else { args = []string{"-C", repopath, "pull", "--all"} } - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), auth, args...) return cmd.Run() } -func (g GitCmd) Fetch(path string) error { +func (g GitCmd) Fetch(path string, auth *Auth) error { _, err := os.Stat(path) if err != nil { return err } args := []string{"-C", path, "fetch", "--all", "--tags"} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), auth, args...) return cmd.Run() } -func (g GitCmd) LFSFetch(path string) error { +func (g GitCmd) LFSFetch(path string, auth *Auth) error { _, err := os.Stat(path) if err != nil { return err } args := []string{"-C", path, "lfs", "fetch", "--all"} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), auth, args...) return cmd.Run() } -func (g GitCmd) MirrorPull(path string) error { +func (g GitCmd) MirrorPull(path string, auth *Auth) error { _, err := os.Stat(path) if err != nil { return err } args := []string{"-C", path, "pull", "--all", "--tags"} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), auth, args...) return cmd.Run() } @@ -81,18 +108,18 @@ func (g GitCmd) NewRemote(name, url, path string) error { return err } args := []string{"-C", path, "remote", "add", name, url} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), nil, args...) return cmd.Run() } -func (g GitCmd) Push(path, remote string) error { +func (g GitCmd) Push(path, remote string, auth *Auth) error { _, err := os.Stat(path) if err != nil { return err } args := []string{"-C", path, "push", "--all", remote} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), auth, args...) output, err := cmd.CombinedOutput() if err != nil { @@ -111,7 +138,7 @@ func (g GitCmd) Checkout(path, branch string) error { return err } args := []string{"checkout", branch} - cmd := exec.CommandContext(context.Background(), g.CMD, args...) + cmd := g.Command(context.Background(), nil, args...) output, err := cmd.CombinedOutput() if err != nil { @@ -125,10 +152,21 @@ func (g GitCmd) Checkout(path, branch string) error { } func (g GitCmd) SSHPush(path, remote, key string) error { - err := os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -i %s", key)) + _, err := os.Stat(path) if err != nil { return err } + args := []string{"-C", path, "push", "--all", remote} + cmd := g.Command(context.Background(), nil, args...) + cmd.Env = append(os.Environ(), fmt.Sprintf("GIT_SSH_COMMAND=ssh -i %s", key)) + + output, err := cmd.CombinedOutput() + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return fmt.Errorf("%s", strings.TrimSuffix(string(output), "\n")) + } + } - return g.Push(path, remote) + return err } diff --git a/gitcmd/gitcmd_test.go b/gitcmd/gitcmd_test.go new file mode 100644 index 00000000..1ada2a44 --- /dev/null +++ b/gitcmd/gitcmd_test.go @@ -0,0 +1,284 @@ +package gitcmd + +import ( + "context" + "encoding/base64" + "fmt" + "reflect" + "strings" + "testing" +) + +func TestAuth_Env_Nil(t *testing.T) { + t.Parallel() + + var auth *Auth + if got := auth.Env(); got != nil { + t.Fatalf("expected nil env for nil Auth, got %v", got) + } +} + +func TestAuth_Env_Empty(t *testing.T) { + t.Parallel() + + auth := &Auth{Username: "", Password: ""} + if got := auth.Env(); got != nil { + t.Fatalf("expected nil env for empty Auth, got %v", got) + } +} + +func TestAuth_Env_GitHubApp(t *testing.T) { + t.Parallel() + + auth := &Auth{ + Username: "x-access-token", + Password: "test-token", + } + + env := auth.Env() + expectedHeader := "Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte("x-access-token:test-token")) + expectedEnv := []string{ + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=http.extraHeader", + "GIT_CONFIG_VALUE_0=" + expectedHeader, + } + + if !reflect.DeepEqual(env, expectedEnv) { + t.Fatalf("auth.Env() = %v, want %v", env, expectedEnv) + } +} + +func TestAuth_Env_UserPassword(t *testing.T) { + t.Parallel() + + auth := &Auth{ + Username: "octocat", + Password: "secretpassword456", + } + + env := auth.Env() + expectedHeader := "Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte("octocat:secretpassword456")) + expectedEnv := []string{ + "GIT_CONFIG_COUNT=1", + "GIT_CONFIG_KEY_0=http.extraHeader", + "GIT_CONFIG_VALUE_0=" + expectedHeader, + } + + if !reflect.DeepEqual(env, expectedEnv) { + t.Fatalf("auth.Env() = %v, want %v", env, expectedEnv) + } +} + +func TestGitCmd_Command_WithoutAuth(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + cmd := g.Command(context.Background(), nil, "status") + + if cmd.Path != "git" && !strings.HasSuffix(cmd.Path, "git") && !strings.HasSuffix(cmd.Path, "git.exe") { + t.Errorf("cmd.Path = %q, want 'git'", cmd.Path) + } + if !reflect.DeepEqual(cmd.Args, []string{"git", "status"}) { + t.Errorf("cmd.Args = %v, want ['git', 'status']", cmd.Args) + } + if cmd.Env != nil { + t.Errorf("cmd.Env for nil Auth = %v, want nil", cmd.Env) + } +} + +func TestGitCmd_Command_WithAuth(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + auth := &Auth{Username: "x-access-token", Password: "test-token"} + cmd := g.Command(context.Background(), auth, "clone", "https://github.com/owner/repo.git", "/tmp/repo") + + if cmd.Env == nil { + t.Fatal("cmd.Env is nil, want environment variables with auth config") + } + + hasConfigCount := false + hasConfigKey := false + hasConfigValue := false + + expectedValue := fmt.Sprintf("GIT_CONFIG_VALUE_0=Authorization: Basic %s", + base64.StdEncoding.EncodeToString([]byte("x-access-token:test-token"))) + + for _, e := range cmd.Env { + if e == "GIT_CONFIG_COUNT=1" { + hasConfigCount = true + } + if e == "GIT_CONFIG_KEY_0=http.extraHeader" { + hasConfigKey = true + } + if e == expectedValue { + hasConfigValue = true + } + } + + if !hasConfigCount || !hasConfigKey || !hasConfigValue { + t.Errorf("cmd.Env missing required GIT_CONFIG variables, env = %v", cmd.Env) + } + + // Verify the URL in cmd.Args does NOT contain token + for _, arg := range cmd.Args { + if strings.Contains(arg, "test-token") { + t.Errorf("token leaked into command arguments: %q", arg) + } + } +} + +func TestGitCmd_CloneCommand(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + url string + reponame string + bare bool + mirror bool + auth *Auth + wantArgs []string + wantAuth bool + }{ + { + name: "regular clone without auth", + url: "https://github.com/owner/repo.git", + reponame: "/tmp/repo", + bare: false, + mirror: false, + auth: nil, + wantArgs: []string{"git", "clone", "https://github.com/owner/repo.git", "/tmp/repo"}, + wantAuth: false, + }, + { + name: "bare clone with token", + url: "https://github.com/owner/repo.git", + reponame: "/tmp/repo.git", + bare: true, + mirror: false, + auth: &Auth{Username: "x-access-token", Password: "my-token"}, + wantArgs: []string{"git", "clone", "https://github.com/owner/repo.git", "/tmp/repo.git", "--bare"}, + wantAuth: true, + }, + { + name: "mirror clone with user/pass", + url: "https://github.com/owner/repo.git", + reponame: "/tmp/repo.git", + bare: true, + mirror: true, + auth: &Auth{Username: "user", Password: "pass"}, + wantArgs: []string{"git", "clone", "https://github.com/owner/repo.git", "/tmp/repo.git", "--bare", "--mirror"}, + wantAuth: true, + }, + } + + g := GitCmd{CMD: "git"} + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + args := []string{"clone", tt.url, tt.reponame} + if tt.bare { + args = append(args, "--bare") + } + if tt.mirror { + args = append(args, "--mirror") + } + cmd := g.Command(context.Background(), tt.auth, args...) + + if !reflect.DeepEqual(cmd.Args, tt.wantArgs) { + t.Errorf("cmd.Args = %v, want %v", cmd.Args, tt.wantArgs) + } + + if tt.wantAuth && cmd.Env == nil { + t.Error("cmd.Env is nil, want auth environment") + } + if !tt.wantAuth && cmd.Env != nil { + t.Errorf("cmd.Env = %v, want nil", cmd.Env) + } + + for _, arg := range cmd.Args { + if strings.Contains(arg, "@") { + t.Errorf("URL in arguments contains credentials: %s", arg) + } + } + }) + } +} + +func TestGitCmd_PullCommand(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + auth := &Auth{Username: "x-access-token", Password: "test-token"} + + // Bare/mirror fetch + bareArgs := []string{"-C", "/tmp/repo.git", "fetch", "--all"} + bareCmd := g.Command(context.Background(), auth, bareArgs...) + wantBareArgs := []string{"git", "-C", "/tmp/repo.git", "fetch", "--all"} + if !reflect.DeepEqual(bareCmd.Args, wantBareArgs) { + t.Errorf("bareCmd.Args = %v, want %v", bareCmd.Args, wantBareArgs) + } + if bareCmd.Env == nil { + t.Error("bareCmd.Env is nil, want auth environment") + } + + // Non-bare pull + pullArgs := []string{"-C", "/tmp/repo", "pull", "--all"} + pullCmd := g.Command(context.Background(), auth, pullArgs...) + wantPullArgs := []string{"git", "-C", "/tmp/repo", "pull", "--all"} + if !reflect.DeepEqual(pullCmd.Args, wantPullArgs) { + t.Errorf("pullCmd.Args = %v, want %v", pullCmd.Args, wantPullArgs) + } + if pullCmd.Env == nil { + t.Error("pullCmd.Env is nil, want auth environment") + } +} + +func TestGitCmd_LFSFetchCommand(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + auth := &Auth{Username: "x-access-token", Password: "test-token"} + + lfsArgs := []string{"-C", "/tmp/repo.git", "lfs", "fetch", "--all"} + cmd := g.Command(context.Background(), auth, lfsArgs...) + wantArgs := []string{"git", "-C", "/tmp/repo.git", "lfs", "fetch", "--all"} + + if !reflect.DeepEqual(cmd.Args, wantArgs) { + t.Errorf("cmd.Args = %v, want %v", cmd.Args, wantArgs) + } + if cmd.Env == nil { + t.Fatal("cmd.Env is nil, want auth environment for LFS fetch") + } + + foundExtraHeader := false + for _, e := range cmd.Env { + if e == "GIT_CONFIG_KEY_0=http.extraHeader" { + foundExtraHeader = true + } + } + if !foundExtraHeader { + t.Error("cmd.Env does not contain GIT_CONFIG_KEY_0=http.extraHeader for LFS fetch") + } +} + +func TestGitCmd_LocalCommandsHaveNoAuth(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + + // Checkout should have no auth + checkoutCmd := g.Command(context.Background(), nil, "checkout", "main") + if checkoutCmd.Env != nil { + t.Errorf("checkoutCmd.Env = %v, want nil", checkoutCmd.Env) + } + + // NewRemote should have no auth + remoteCmd := g.Command(context.Background(), nil, "-C", "/tmp/repo", "remote", "add", "origin", "https://github.com/owner/repo.git") + if remoteCmd.Env != nil { + t.Errorf("remoteCmd.Env = %v, want nil", remoteCmd.Env) + } +} diff --git a/local/local.go b/local/local.go index 27d1236a..183f53f2 100644 --- a/local/local.go +++ b/local/local.go @@ -51,6 +51,16 @@ func tokenAuth(repo types.Repo) *http.BasicAuth { } } +func toGitCmdAuth(auth transport.AuthMethod) *gitcmd.Auth { + if basicAuth, ok := auth.(*http.BasicAuth); ok && basicAuth != nil { + return &gitcmd.Auth{ + Username: basicAuth.Username, + Password: basicAuth.Password, + } + } + return nil +} + // Locally TODO. func Locally(repo types.Repo, l types.Local, dry bool) bool { sub = logger.CreateSubLogger("stage", "locally", "path", l.Path) @@ -187,7 +197,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { sub.Info(). Msgf("opening %s locally", types.Green(repo.Name)) - err := updateRepository(repo.Name, auth, dry, l) + err := updateRepository(repo, auth, dry, l) if err != nil { switch { case errors.Is(err, git.NoErrAlreadyUpToDate): @@ -331,32 +341,63 @@ func pruneOldBackups(parentdir string, repoName string, l types.Local) error { return nil } -func updateRepository(reponame string, auth transport.AuthMethod, dry bool, l types.Local) error { - r, err := git.PlainOpen(filepath.Join(l.Path, reponame)) +func sanitizeRemote(r *git.Repository, cleanURL string) error { + if cleanURL == "" { + return nil + } + cfg, err := r.Config() + if err != nil { + return err + } + rem, ok := cfg.Remotes["origin"] + if !ok || rem == nil { + return nil + } + changed := false + for i, u := range rem.URLs { + if (strings.HasPrefix(u, "http://") || strings.HasPrefix(u, "https://")) && strings.Contains(u, "@") { + rem.URLs[i] = cleanURL + changed = true + } + } + if changed { + return r.SetConfig(cfg) + } + return nil +} + +func updateRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l types.Local) error { + r, err := git.PlainOpen(filepath.Join(l.Path, repo.Name)) if err != nil { return err } if !dry { if l.LFS { - _, err = os.Stat(filepath.Join(l.Path, reponame)) + _, err = os.Stat(filepath.Join(l.Path, repo.Name)) if err != nil { return err } + if !repo.Origin.SSH && repo.URL != "" { + if err := sanitizeRemote(r, repo.URL); err != nil { + sub.Warn().Str("repo", repo.Name).Msgf("could not sanitize remote: %s", err) + } + } + sub.Info(). - Msgf("pulling %s", types.Green(reponame)) + Msgf("pulling %s", types.Green(repo.Name)) - err = gitc.Pull(l.Bare, l.Mirror, filepath.Join(l.Path, reponame)) + err = gitc.Pull(l.Bare, l.Mirror, filepath.Join(l.Path, repo.Name), toGitCmdAuth(auth)) if err != nil { return err } if l.Bare || l.Mirror { sub.Info(). - Msgf("fetching lfs files for %s", types.Green(reponame)) + Msgf("fetching lfs files for %s", types.Green(repo.Name)) - err = gitc.LFSFetch(filepath.Join(l.Path, reponame)) + err = gitc.LFSFetch(filepath.Join(l.Path, repo.Name), toGitCmdAuth(auth)) if err != nil { return err } @@ -368,7 +409,7 @@ func updateRepository(reponame string, auth transport.AuthMethod, dry bool, l ty return err } sub.Info(). - Msgf("pulling %s", types.Green(reponame)) + Msgf("pulling %s", types.Green(repo.Name)) if !l.Bare && !l.Mirror { w, worktreeErr := r.Worktree() if worktreeErr != nil && !errors.Is(worktreeErr, git.NoErrAlreadyUpToDate) { @@ -428,25 +469,7 @@ func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l typ } if l.LFS { - if repo.Token != "" { - if strings.HasPrefix(url, "http://") { - url = strings.ReplaceAll(url, "http://", fmt.Sprintf("http://xyz:%s@", repo.Token)) - } - - if strings.HasPrefix(url, "https://") { - url = strings.ReplaceAll(url, "https://", fmt.Sprintf("https://xyz:%s@", repo.Token)) - } - } else if repo.Origin.Username != "" && repo.Origin.Password != "" { - if strings.HasPrefix(url, "http://") { - url = strings.ReplaceAll(url, "http://", fmt.Sprintf("http://%s:%s@", repo.Origin.Username, repo.Origin.Password)) - } - - if strings.HasPrefix(url, "https://") { - url = strings.ReplaceAll(url, "https://", fmt.Sprintf("https://%s:%s@", repo.Origin.Username, repo.Origin.Password)) - } - } - - err = gitc.Clone(url, filepath.Join(l.Path, repo.Name), l.Bare, l.Mirror) + err = gitc.Clone(url, filepath.Join(l.Path, repo.Name), l.Bare, l.Mirror, toGitCmdAuth(auth)) if err != nil { return err } @@ -455,7 +478,7 @@ func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l typ sub.Info(). Msgf("fetching lfs files for %s", types.Green(repo.Name)) - err = gitc.LFSFetch(filepath.Join(l.Path, repo.Name)) + err = gitc.LFSFetch(filepath.Join(l.Path, repo.Name), toGitCmdAuth(auth)) if err != nil { return err } @@ -538,6 +561,11 @@ func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repositor var auth transport.AuthMethod if repo.Token != "" { auth = tokenAuth(repo) + } else if repo.Origin.Username != "" && repo.Origin.Password != "" { + auth = &http.BasicAuth{ + Username: repo.Origin.Username, + Password: repo.Origin.Password, + } } if repo.Origin.LFS { g, err := gitcmd.New() @@ -546,15 +574,7 @@ func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repositor } gitc = g - if strings.HasPrefix(repo.URL, "http://") { - repo.URL = strings.ReplaceAll(repo.URL, "http://", fmt.Sprintf("http://xyz:%s@", repo.Token)) - } - - if strings.HasPrefix(repo.URL, "https://") { - repo.URL = strings.ReplaceAll(repo.URL, "https://", fmt.Sprintf("https://xyz:%s@", repo.Token)) - } - - err = gitc.Clone(repo.URL, tempdir, isBare, false) + err = gitc.Clone(repo.URL, tempdir, isBare, false, toGitCmdAuth(auth)) if err != nil { return nil, err } @@ -600,7 +620,7 @@ func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repositor return nil, err } - err = gitc.MirrorPull(tempdir) + err = gitc.MirrorPull(tempdir, toGitCmdAuth(auth)) if err != nil { return nil, err } @@ -691,20 +711,12 @@ func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url strin return err } } else { - if strings.HasPrefix(url, "http://") { - url = strings.ReplaceAll(url, "http://", fmt.Sprintf("http://xyz:%s@", token)) - } - - if strings.HasPrefix(url, "https://") { - url = strings.ReplaceAll(url, "https://", fmt.Sprintf("https://xyz:%s@", token)) - } - err = gitc.NewRemote(remote, url, worktree.Filesystem.Root()) if err != nil { return err } - err = gitc.Push(worktree.Filesystem.Root(), remote) + err = gitc.Push(worktree.Filesystem.Root(), remote, toGitCmdAuth(auth)) if err != nil { return err } diff --git a/local/local_test.go b/local/local_test.go index fb10515a..66831655 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -9,7 +9,10 @@ import ( "github.com/cooperspencer/gickup/types" "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/go-git/go-git/v5/storage/memory" ) @@ -184,6 +187,191 @@ func TestTokenAuth_WithTokenUser(t *testing.T) { } } +func TestToGitCmdAuth_Nil(t *testing.T) { + t.Parallel() + + if got := toGitCmdAuth(nil); got != nil { + t.Fatalf("toGitCmdAuth(nil) = %v, want nil", got) + } +} + +func TestToGitCmdAuth_FromTokenAuth_GitHubApp(t *testing.T) { + t.Parallel() + + repo := types.Repo{ + Token: "ghs_testtoken123", + NoTokenUser: true, + } + auth := tokenAuth(repo) + gitAuth := toGitCmdAuth(auth) + if gitAuth == nil { + t.Fatal("toGitCmdAuth returned nil, want *gitcmd.Auth") + } + if gitAuth.Username != "x-access-token" { + t.Errorf("Username = %q, want %q", gitAuth.Username, "x-access-token") + } + if gitAuth.Password != "ghs_testtoken123" { + t.Errorf("Password = %q, want %q", gitAuth.Password, "ghs_testtoken123") + } + + env := gitAuth.Env() + if len(env) != 3 { + t.Fatalf("expected 3 env vars, got %d: %v", len(env), env) + } + if env[0] != "GIT_CONFIG_COUNT=1" || env[1] != "GIT_CONFIG_KEY_0=http.extraHeader" { + t.Errorf("unexpected env vars: %v", env) + } + if !strings.HasPrefix(env[2], "GIT_CONFIG_VALUE_0=Authorization: Basic ") { + t.Errorf("unexpected GIT_CONFIG_VALUE_0: %s", env[2]) + } +} + +func TestToGitCmdAuth_FromUsernamePassword(t *testing.T) { + t.Parallel() + + auth := &http.BasicAuth{ + Username: "myuser", + Password: "mypassword", + } + gitAuth := toGitCmdAuth(auth) + if gitAuth == nil { + t.Fatal("toGitCmdAuth returned nil, want *gitcmd.Auth") + } + if gitAuth.Username != "myuser" { + t.Errorf("Username = %q, want %q", gitAuth.Username, "myuser") + } + if gitAuth.Password != "mypassword" { + t.Errorf("Password = %q, want %q", gitAuth.Password, "mypassword") + } +} + +func TestToGitCmdAuth_SSHAuth(t *testing.T) { + t.Parallel() + + auth := &ssh.PublicKeys{} + if got := toGitCmdAuth(auth); got != nil { + t.Fatalf("toGitCmdAuth(ssh) = %v, want nil", got) + } +} + +func TestSanitizeRemote_RewritesCredentialURL(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + r, err := git.PlainInit(dir, true) + if err != nil { + t.Fatalf("failed to init repo: %v", err) + } + + _, err = r.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"https://xyz:expired-token-123@github.com/owner/repo.git"}, + }) + if err != nil { + t.Fatalf("failed to create remote: %v", err) + } + + err = sanitizeRemote(r, "https://github.com/owner/repo.git") + if err != nil { + t.Fatalf("sanitizeRemote returned error: %v", err) + } + + // Reopen the repository from disk to verify persistence + reopened, err := git.PlainOpen(dir) + if err != nil { + t.Fatalf("failed to reopen repo: %v", err) + } + + cfg, err := reopened.Config() + if err != nil { + t.Fatalf("failed to get config: %v", err) + } + + rem, ok := cfg.Remotes["origin"] + if !ok { + t.Fatal("remote origin not found") + } + + wantURLs := []string{"https://github.com/owner/repo.git"} + if !reflect.DeepEqual(rem.URLs, wantURLs) { + t.Errorf("remote URLs = %v, want %v", rem.URLs, wantURLs) + } + + for _, u := range rem.URLs { + if strings.Contains(u, "expired-token-123") || strings.Contains(u, "xyz:") || strings.Contains(u, "@") { + t.Errorf("token or credentials leaked in remote URL: %s", u) + } + } +} + +func TestSanitizeRemote_PreservesSSH(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + r, err := git.PlainInit(dir, true) + if err != nil { + t.Fatalf("failed to init repo: %v", err) + } + + _, err = r.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"git@github.com:owner/repo.git"}, + }) + if err != nil { + t.Fatalf("failed to create remote: %v", err) + } + + err = sanitizeRemote(r, "https://github.com/owner/repo.git") + if err != nil { + t.Fatalf("sanitizeRemote returned error: %v", err) + } + + cfg, err := r.Config() + if err != nil { + t.Fatalf("failed to get config: %v", err) + } + + rem := cfg.Remotes["origin"] + wantURLs := []string{"git@github.com:owner/repo.git"} + if !reflect.DeepEqual(rem.URLs, wantURLs) { + t.Errorf("SSH remote URLs = %v, want %v", rem.URLs, wantURLs) + } +} + +func TestSanitizeRemote_PreservesCleanURL(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + r, err := git.PlainInit(dir, true) + if err != nil { + t.Fatalf("failed to init repo: %v", err) + } + + _, err = r.CreateRemote(&config.RemoteConfig{ + Name: "origin", + URLs: []string{"https://github.com/owner/repo.git"}, + }) + if err != nil { + t.Fatalf("failed to create remote: %v", err) + } + + err = sanitizeRemote(r, "https://github.com/owner/repo.git") + if err != nil { + t.Fatalf("sanitizeRemote returned error: %v", err) + } + + cfg, err := r.Config() + if err != nil { + t.Fatalf("failed to get config: %v", err) + } + + rem := cfg.Remotes["origin"] + wantURLs := []string{"https://github.com/owner/repo.git"} + if !reflect.DeepEqual(rem.URLs, wantURLs) { + t.Errorf("clean remote URLs = %v, want %v", rem.URLs, wantURLs) + } +} + func TestGetPushRefSpecs(t *testing.T) { t.Parallel() From 002be7a6e850624c755777ae145fe5577fabd6a6 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Fri, 28 Aug 2026 15:52:10 +0200 Subject: [PATCH 27/44] fixed lfs partial backup (#426) --- local/local.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/local/local.go b/local/local.go index 183f53f2..fb42f866 100644 --- a/local/local.go +++ b/local/local.go @@ -588,8 +588,20 @@ func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repositor Auth: auth, Force: true, }) - if errors.Is(err, git.NoErrAlreadyUpToDate) { - return r, nil + if err != nil && !errors.Is(err, git.NoErrAlreadyUpToDate) { + return r, err + } + + // A regular clone only downloads the LFS objects needed for the + // checked-out branch. Without fetching the rest, objects referenced + // by other branches/tags or older commits are missing locally, + // which makes the later `git push --all` fail with + // "missing or corrupt local objects". + sub.Info(). + Msgf("fetching lfs files for %s", types.Green(repo.Name)) + + if lfsErr := gitc.LFSFetch(tempdir, toGitCmdAuth(auth)); lfsErr != nil { + return r, lfsErr } // Get the symbolic reference for HEAD From cba438309e1986940ba66ec36b8a8b83638a9109 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Fri, 28 Aug 2026 16:18:36 +0200 Subject: [PATCH 28/44] add a backdown for gists too (#427) --- github/github.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/github/github.go b/github/github.go index 8e983a39..c2e60935 100644 --- a/github/github.go +++ b/github/github.go @@ -443,14 +443,28 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if repo.Gists { gistlistoptions := &github.GistListOptions{ListOptions: github.ListOptions{PerPage: 50}} i := 1 + errorcount := 0 for { gistlistoptions.Page = i gists, _, err := client.Gists.List(context.Background(), repo.User, gistlistoptions) if err != nil { sub.Error(). Msg(err.Error()) - continue + var rateLimitErr *github.RateLimitError + var abuseRateLimitErr *github.AbuseRateLimitError + if errors.As(err, &rateLimitErr) || errors.As(err, &abuseRateLimitErr) { + sub.Warn().Msg("wait for one hour.") + time.Sleep(1 * time.Hour) + continue + } + if errorcount < 5 { + errorcount++ + time.Sleep(5 * time.Second) + continue + } + break } + errorcount = 0 if len(gists) == 0 { break } From 03eae3beb38c5b37e4cb38fa689e0b7556aaf3e7 Mon Sep 17 00:00:00 2001 From: Ryan Date: Sun, 30 Aug 2026 22:10:48 +0800 Subject: [PATCH 29/44] Add Scoop to distribution packages in README (#428) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 2b9c8b74..3d6cf573 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ https://github.com/cooperspencer/gickup/discussions |Arch|[gickup](https://aur.archlinux.org/packages/gickup/)|[me](https://github.com/cooperspencer)| |Homebrew|[gickup](https://formulae.brew.sh/formula/gickup#default)|| |Fedora|[gickup](https://copr.fedorainfracloud.org/coprs/frostyx/gickup/)|[FrostyX](https://github.com/FrostyX)| +|Scoop|[gickup](https://scoop.sh/#/apps?q=gickup&id=493e856707843828b4491004edfb6ddedc4bd1e1)|| ## Issues The mirroring to Gitlab doesn't work, or at least I can't test it properly because I have no access to a Gitlab EE instance. From 1774a02416a7a5ed47976d36057b3bbaa0ad9c5c Mon Sep 17 00:00:00 2001 From: Om Lahore Date: Tue, 1 Sep 2026 13:41:28 +0530 Subject: [PATCH 30/44] Check the error when listing GitLab descendant groups (#429) Both ScanAndCollect calls dropped their error. ScanAndCollect returns whatever it managed to collect alongside the error, so a failed or partial ListDescendantGroups left subgroups short without saying so, and the include/exclude org maps were built from an incomplete list. For includeorgs that silently drops repos from the backup set. The GetGroup call seven lines above each of these already logs and continues on error, so this matches the surrounding code. --- gitlab/gitlab.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index c32d8b1f..56db880d 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -221,6 +221,11 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { subgroups, err := gitlab.ScanAndCollect(func(p gitlab.PaginationOptionFunc) ([]*gitlab.Group, *gitlab.Response, error) { return client.Groups.ListDescendantGroups(group.ID, &gitlab.ListDescendantGroupsOptions{}, p) }) + if err != nil { + sub.Error(). + Msg(err.Error()) + continue + } for _, sub := range subgroups { includeorgs[sub.FullPath] = true @@ -238,6 +243,11 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { subgroups, err := gitlab.ScanAndCollect(func(p gitlab.PaginationOptionFunc) ([]*gitlab.Group, *gitlab.Response, error) { return client.Groups.ListDescendantGroups(group.ID, &gitlab.ListDescendantGroupsOptions{}, p) }) + if err != nil { + sub.Error(). + Msg(err.Error()) + continue + } for _, sub := range subgroups { excludeorgs[sub.FullPath] = true From 11eb3ff76b2b5303a101d02751a5884c3d5142af Mon Sep 17 00:00:00 2001 From: Ben Burwood <77846694+ben-burwood@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:05:42 +0100 Subject: [PATCH 31/44] Implement OpenGist Source (#430) * Implement OpenGist Source * Fix Lint Issues * Fix gitlab test lint error * global gickup UserAgent * Fix package name lint * fix go package name --- README.md | 1 + conf.example.yml | 13 ++ gickup_spec.json | 30 +++++ httpclient/useragent.go | 37 ++++++ httpclient/useragent_test.go | 29 +++++ main.go | 11 ++ opengist/opengist.go | 224 +++++++++++++++++++++++++++++++++++ opengist/opengist_test.go | 180 ++++++++++++++++++++++++++++ types/types.go | 2 + 9 files changed, 527 insertions(+) create mode 100644 httpclient/useragent.go create mode 100644 httpclient/useragent_test.go create mode 100644 opengist/opengist.go create mode 100644 opengist/opengist_test.go diff --git a/README.md b/README.md index 3d6cf573..40add751 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ You can clone/mirror repositories from: - Bitbucket - OneDev - Sourcehut +- Opengist - Any You can clone/mirror repositories to: diff --git a/conf.example.yml b/conf.example.yml index 028e3663..cc7478ab 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -212,6 +212,19 @@ source: wiki: true # includes wiki too filter: lastactivity: 1y # only clone repos which had activity during the last year + opengist: + - url: http(s)://url-to-opengist # required, base url of your Opengist instance, e.g. https://gist.example.com + token: some-token # Opengist personal access token (og_...); used for the API and to clone private gists + # token_file: token.txt # alternatively, specify token in a file + # user: some-user # optional: back up this user's gists (GET /api/users//gists). + # leave user out and set a token to back up the token owner's own gists (GET /api/gists). + # leave out both user and token to back up only public gists (GET /api/gists/public). + # note: Opengist's git-over-http must be enabled on the server to clone (OG_HTTP_GIT_ENABLED=true). + exclude: # this excludes the gists with these slugs/ids + - foo + - bar + include: # this only includes the gists with these slugs/ids + - foobar any: - url: url-to-any-repo # can be https, http or ssh user: your-preferred-user # the user to want to associate with this repo, default: git diff --git a/gickup_spec.json b/gickup_spec.json index 1200bd10..ab9600fe 100644 --- a/gickup_spec.json +++ b/gickup_spec.json @@ -449,6 +449,36 @@ "additionalProperties": false } }, + "opengist": { + "type": "array", + "items": { + "type": "object", + "properties": { + "token": { + "$ref": "#/definitions/source/properties/token" + }, + "token_file": { + "$ref": "#/definitions/source/properties/token_file" + }, + "url": { + "$ref": "#/definitions/source/properties/url" + }, + "user": { + "$ref": "#/definitions/source/properties/user" + }, + "exclude": { + "$ref": "#/definitions/source/properties/exclude" + }, + "include": { + "$ref": "#/definitions/source/properties/include" + } + }, + "additionalProperties": false + }, + "required": [ + "url" + ] + }, "any": { "type": "array", "items": { diff --git a/httpclient/useragent.go b/httpclient/useragent.go new file mode 100644 index 00000000..6ebcd9be --- /dev/null +++ b/httpclient/useragent.go @@ -0,0 +1,37 @@ +package httpclient + +import ( + "net/http" + "sync" + + gitclient "github.com/go-git/go-git/v5/plumbing/transport/client" + githttp "github.com/go-git/go-git/v5/plumbing/transport/http" +) + +// Custom UserAgent for tracking gickup calls in the network +// Some Services may only serve git-over-HTTP endpoints to clients whose User-Agent starts with "git/" +const gickupUserAgent = "git/gickup" + +type userAgentTransport struct { + base http.RoundTripper +} + +func (t userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) { + clone := req.Clone(req.Context()) + clone.Header.Set("User-Agent", gickupUserAgent) + return t.base.RoundTrip(clone) +} + +// Run once sync used to gate init of the UserAgent for go-git +var installUserAgentOnce sync.Once + +// ApplyGickupUserAgent installs go-git HTTP(S) clients that send a git-compatible User-Agent. +// "git/" User-Agent is what the real git client sends, so it is accepted by every other hoster. +func ApplyGickupUserAgent() { + installUserAgentOnce.Do(func() { + hc := &http.Client{Transport: userAgentTransport{base: http.DefaultTransport}} + c := githttp.NewClient(hc) + gitclient.InstallProtocol("http", c) + gitclient.InstallProtocol("https", c) + }) +} diff --git a/httpclient/useragent_test.go b/httpclient/useragent_test.go new file mode 100644 index 00000000..22af9919 --- /dev/null +++ b/httpclient/useragent_test.go @@ -0,0 +1,29 @@ +package httpclient + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestUserAgentTransportRewrites(t *testing.T) { + t.Parallel() + var got string + srv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + got = r.Header.Get("User-Agent") + })) + defer srv.Close() + + rt := userAgentTransport{base: http.DefaultTransport} + req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, srv.URL, nil) + req.Header.Set("User-Agent", "go-git/5.x") + resp, err := rt.RoundTrip(req) + if err != nil { + t.Fatalf("roundtrip: %v", err) + } + resp.Body.Close() + + if got != gickupUserAgent { + t.Fatalf("expected User-Agent %q, got %q", gickupUserAgent, got) + } +} diff --git a/main.go b/main.go index c64b3a71..3476d757 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( "github.com/cooperspencer/gickup/github" "github.com/cooperspencer/gickup/gitlab" "github.com/cooperspencer/gickup/gogs" + "github.com/cooperspencer/gickup/httpclient" "github.com/cooperspencer/gickup/local" "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/metrics/apprise" @@ -28,6 +29,7 @@ import ( "github.com/cooperspencer/gickup/metrics/ntfy" "github.com/cooperspencer/gickup/metrics/prometheus" "github.com/cooperspencer/gickup/onedev" + "github.com/cooperspencer/gickup/opengist" "github.com/cooperspencer/gickup/radicle" "github.com/cooperspencer/gickup/s3" "github.com/cooperspencer/gickup/sourcehut" @@ -120,6 +122,7 @@ func expandConfigPaths(c *types.Conf) { expandGenRepoPaths(c.Source.BitBucket) expandGenRepoPaths(c.Source.OneDev) expandGenRepoPaths(c.Source.Sourcehut) + expandGenRepoPaths(c.Source.Opengist) expandGenRepoPaths(c.Source.Any) expandGenRepoPaths(c.Destination.Gitlab) @@ -1075,6 +1078,8 @@ func runBackup(conf *types.Conf, num int) { prometheus.JobsStarted.Inc() + httpclient.ApplyGickupUserAgent() + // Github repos, ran := github.Get(conf) if ran { @@ -1127,6 +1132,12 @@ func runBackup(conf *types.Conf, num int) { } backup(repos, conf) + repos, ran = opengist.Get(conf) + if ran { + prometheus.CountReposDiscovered.WithLabelValues("opengist", numstring).Set(float64(len(repos))) + } + backup(repos, conf) + endTime := time.Now() duration := endTime.Sub(startTime) diff --git a/opengist/opengist.go b/opengist/opengist.go new file mode 100644 index 00000000..ffa6e6fb --- /dev/null +++ b/opengist/opengist.go @@ -0,0 +1,224 @@ +package opengist + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/cooperspencer/gickup/logger" + "github.com/cooperspencer/gickup/types" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" +) + +var sub zerolog.Logger + +// perPage is the page size used when paginating the Opengist API. +const perPage = 50 + +// gist mirrors the relevant fields of Opengist's GistSimple API object +// (internal/web/handlers/api/v1/types/gist.go upstream). +type gist struct { + ID string `json:"id"` + SlugURL string `json:"slug_url"` + Owner user `json:"owner"` + Title string `json:"title"` + Description string `json:"description"` + Public bool `json:"public"` + Visibility string `json:"visibility"` + CloneURL string `json:"clone_url"` + SSHURL string `json:"ssh_url"` +} + +// user mirrors the fields of Opengist's SimpleUser/PrivateUser. +type user struct { + Username string `json:"username"` +} + +// client is a minimal Opengist REST client for the API (Bearer token auth) over net/http. +type client struct { + base string + token string + http *http.Client +} + +func (c *client) get(path string, query url.Values) ([]byte, error) { + endpoint := c.base + path + if len(query) > 0 { + endpoint += "?" + query.Encode() + } + + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, endpoint, nil) + if err != nil { + return nil, err + } + if c.token != "" { + req.Header.Set("Authorization", "Bearer "+c.token) + } + + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, fmt.Errorf("%s returned %d: %s", endpoint, resp.StatusCode, strings.TrimSpace(string(body))) + } + + return body, nil +} + +// getUser returns the token owner (GET /api/user). +func (c *client) getUser() (user, error) { + var u user + body, err := c.get("/api/user", nil) + if err != nil { + return u, err + } + return u, json.Unmarshal(body, &u) +} + +// listGists pages through a gist-listing endpoint and returns everything. +func (c *client) listGists(path string) ([]gist, error) { + all := []gist{} + for page := 1; ; page++ { + query := url.Values{} + query.Set("page", strconv.Itoa(page)) + query.Set("per_page", strconv.Itoa(perPage)) + + body, err := c.get(path, query) + if err != nil { + return nil, err + } + + batch := []gist{} + if err := json.Unmarshal(body, &batch); err != nil { + return nil, err + } + + all = append(all, batch...) + + if len(batch) < perPage { + break + } + } + return all, nil +} + +// Get lists gists from every configured Opengist source and +// turns them into repositories gickup can clone and back up. +// +// Listing endpoint is chosen per source entry: +// - user set -> GET /api/users/{user}/gists (user's gists) +// - user empty + token -> GET /api/gists (token owner's own gists) +// - user empty + no token -> GET /api/gists/public (public gists only) +// +// Opengist's git-over-HTTP basic auth requires the username to match the token owner, +// so private gists are cloned as that user regardless of which endpoint listed them. +// Public/unlisted gists clone anonymously when no token is set. +func Get(conf *types.Conf) ([]types.Repo, bool) { + ran := false + repos := []types.Repo{} + + for _, repo := range conf.Source.Opengist { + ran = true + + if repo.URL == "" { + log.Error(). + Str("stage", "opengist"). + Msg("no url configured") + continue + } + + base := strings.TrimRight(repo.URL, "/") + sub = logger.CreateSubLogger("stage", "opengist", "url", base) + + token := repo.GetToken() + cl := &client{base: base, token: token, http: &http.Client{Timeout: 30 * time.Second}} + + // The clone username must match the token owner for private gists. + tokenOwner := "" + if token != "" { + u, err := cl.getUser() + if err != nil { + sub.Warn(). + Msgf("couldn't determine token owner, private gists may not clone: %s", err.Error()) + } else { + tokenOwner = u.Username + } + } + + var listPath string + switch { + case repo.User != "": + listPath = "/api/users/" + url.PathEscape(repo.User) + "/gists" + sub.Info(). + Msgf("grabbing gists from %s", repo.User) + case token != "": + listPath = "/api/gists" + sub.Info(). + Msg("grabbing my gists") + default: + listPath = "/api/gists/public" + sub.Info(). + Msg("grabbing public gists") + } + + gists, err := cl.listGists(listPath) + if err != nil { + sub.Error(). + Msg(err.Error()) + continue + } + + include := types.GetMap(repo.Include) + exclude := types.GetMap(repo.Exclude) + + for _, g := range gists { + name := g.SlugURL + if name == "" { + name = g.ID + } + + if exclude[name] { + continue + } + if len(repo.Include) > 0 && !include[name] { + continue + } + + // Origin carries the clone credentials. Opengist wants the token + // owner as the basic-auth username, so override it here. + origin := repo + if tokenOwner != "" { + origin.User = tokenOwner + } + + repos = append(repos, types.Repo{ + Name: name, + URL: g.CloneURL, + SSHURL: g.SSHURL, + Token: token, + Origin: origin, + Owner: g.Owner.Username, + Hoster: types.GetHost(base), + Description: g.Description, + Private: !g.Public, + }) + } + } + + return repos, ran +} diff --git a/opengist/opengist_test.go b/opengist/opengist_test.go new file mode 100644 index 00000000..0df58376 --- /dev/null +++ b/opengist/opengist_test.go @@ -0,0 +1,180 @@ +package opengist + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/cooperspencer/gickup/types" +) + +// gistPayload builds an Opengist-shaped gist JSON object. +func gistPayload(slug, owner string, public bool) map[string]interface{} { + return map[string]interface{}{ + "id": slug, + "slug_url": slug, + "owner": map[string]interface{}{"username": owner}, + "title": slug, + "description": "desc-" + slug, + "public": public, + "visibility": map[bool]string{true: "public", false: "private"}[public], + "clone_url": fmt.Sprintf("http://opengist.example/%s/%s.git", owner, slug), + "ssh_url": fmt.Sprintf("git@opengist.example:%s/%s.git", owner, slug), + } +} + +// newServer returns an httptest server that serves a single page of gists on +// the given path plus a token-owner endpoint, then an empty second page. +func newServer(t *testing.T, path string, gists []map[string]interface{}) *httptest.Server { + t.Helper() + mux := http.NewServeMux() + + mux.HandleFunc("/api/user", func(w http.ResponseWriter, _ *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]interface{}{"username": "tokenowner"}) + }) + + mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) { + page, _ := strconv.Atoi(r.URL.Query().Get("page")) + if page > 1 { + _ = json.NewEncoder(w).Encode([]map[string]interface{}{}) + return + } + _ = json.NewEncoder(w).Encode(gists) + }) + + return httptest.NewServer(mux) +} + +func TestGetOwnGists(t *testing.T) { + t.Parallel() + srv := newServer(t, "/api/gists", []map[string]interface{}{ + gistPayload("alpha", "alice", true), + gistPayload("beta", "alice", false), + }) + defer srv.Close() + + conf := &types.Conf{ + Source: types.Source{ + Opengist: []types.GenRepo{{URL: srv.URL, Token: "og_token"}}, + }, + } + + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + if len(repos) != 2 { + t.Fatalf("expected 2 repos, got %d", len(repos)) + } + + r := repos[0] + if r.Name != "alpha" || r.Owner != "alice" { + t.Fatalf("unexpected repo: %#v", r) + } + if r.URL != "http://opengist.example/alice/alpha.git" { + t.Fatalf("unexpected clone url: %q", r.URL) + } + // Clone auth must use the token owner as username (Opengist requirement). + if r.Origin.User != "tokenowner" { + t.Fatalf("expected clone user tokenowner, got %q", r.Origin.User) + } + if repos[1].Private != true { + t.Fatalf("expected beta to be private") + } +} + +func TestGetPublicGistsAnonymous(t *testing.T) { + t.Parallel() + srv := newServer(t, "/api/gists/public", []map[string]interface{}{ + gistPayload("pub", "bob", true), + }) + defer srv.Close() + + conf := &types.Conf{ + Source: types.Source{ + Opengist: []types.GenRepo{{URL: srv.URL}}, + }, + } + + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + if len(repos) != 1 { + t.Fatalf("expected 1 repo, got %d", len(repos)) + } + if repos[0].Token != "" || repos[0].Origin.User != "" { + t.Fatalf("expected anonymous clone, got token=%q user=%q", repos[0].Token, repos[0].Origin.User) + } +} + +func TestGetUserGists(t *testing.T) { + t.Parallel() + srv := newServer(t, "/api/users/carol/gists", []map[string]interface{}{ + gistPayload("one", "carol", true), + }) + defer srv.Close() + + conf := &types.Conf{ + Source: types.Source{ + Opengist: []types.GenRepo{{URL: srv.URL, Token: "og_token", User: "carol"}}, + }, + } + + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + if len(repos) != 1 || repos[0].Owner != "carol" { + t.Fatalf("unexpected repos: %#v", repos) + } +} + +func TestGetIncludeExclude(t *testing.T) { + t.Parallel() + gists := []map[string]interface{}{ + gistPayload("keep", "alice", true), + gistPayload("drop", "alice", true), + } + + t.Run("include", func(t *testing.T) { + t.Parallel() + srv := newServer(t, "/api/gists", gists) + defer srv.Close() + conf := &types.Conf{Source: types.Source{Opengist: []types.GenRepo{{ + URL: srv.URL, Token: "og_token", Include: []string{"keep"}, + }}}} + repos, _ := Get(conf) + if len(repos) != 1 || repos[0].Name != "keep" { + t.Fatalf("include filter failed: %#v", repos) + } + }) + + t.Run("exclude", func(t *testing.T) { + t.Parallel() + srv := newServer(t, "/api/gists", gists) + defer srv.Close() + conf := &types.Conf{Source: types.Source{Opengist: []types.GenRepo{{ + URL: srv.URL, Token: "og_token", Exclude: []string{"drop"}, + }}}} + repos, _ := Get(conf) + if len(repos) != 1 || repos[0].Name != "keep" { + t.Fatalf("exclude filter failed: %#v", repos) + } + }) +} + +func TestGetSkipsMissingURL(t *testing.T) { + t.Parallel() + conf := &types.Conf{Source: types.Source{Opengist: []types.GenRepo{{}}}} + repos, ran := Get(conf) + if !ran { + t.Fatal("expected adapter to run") + } + if len(repos) != 0 { + t.Fatalf("expected no repositories, got %d", len(repos)) + } +} diff --git a/types/types.go b/types/types.go index b9c6824b..86c3ddf6 100644 --- a/types/types.go +++ b/types/types.go @@ -245,6 +245,7 @@ type Source struct { BitBucket []GenRepo `yaml:"bitbucket"` OneDev []GenRepo `yaml:"onedev"` Sourcehut []GenRepo `yaml:"sourcehut"` + Opengist []GenRepo `yaml:"opengist"` Any []GenRepo `yaml:"any"` } @@ -257,6 +258,7 @@ func (source Source) Count() int { len(source.Gitlab) + len(source.OneDev) + len(source.Sourcehut) + + len(source.Opengist) + len(source.Any) } From 97c0b60a0c62d3976cbfd46c2cee27be2e48dbb5 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Wed, 9 Sep 2026 18:06:21 +0200 Subject: [PATCH 32/44] add failure heartbeats and status-aware notifications (#431) --- conf.example.yml | 2 ++ main.go | 24 +++++++++++++++------- metrics/apprise/apprise.go | 16 ++++++++------- metrics/apprise/apprise_test.go | 4 ++-- metrics/gotify/gotify.go | 4 ++-- metrics/gotify/gotify_test.go | 4 ++-- metrics/heartbeat/heartbeat.go | 18 +++++++++++++--- metrics/heartbeat/heartbeat_test.go | 32 ++++++++++++++++++++++++++++- metrics/ntfy/ntfy.go | 4 ++-- metrics/ntfy/ntfy_test.go | 8 ++++---- types/types.go | 3 ++- 11 files changed, 88 insertions(+), 31 deletions(-) diff --git a/conf.example.yml b/conf.example.yml index cc7478ab..0866cd43 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -369,6 +369,8 @@ metrics: urls: - http(s)://url-to-make-request-to - http(s)://another-url-to-make-request-to + failure_urls: # optional - if the backup run encountered any errors, these URLs are pinged instead of the ones above. Useful with services like healthchecks.io that expose a dedicated failure endpoint (e.g. https://hc-ping.com/uuid/fail) + - http(s)://url-to-ping-on-failure push: ntfy: - url: http(s)://url-to-ntfy/your-topic diff --git a/main.go b/main.go index 3476d757..f70168ba 100644 --- a/main.go +++ b/main.go @@ -1144,14 +1144,24 @@ func runBackup(conf *types.Conf, num int) { prometheus.JobsComplete.Inc() prometheus.JobDuration.Observe(duration.Seconds()) - if len(conf.Metrics.Heartbeat.URLs) > 0 { - heartbeat.Send(conf.Metrics.Heartbeat) + exitCode := logger.GetExitCode() + success := exitCode == 0 + + title := "Backup done" + msg := fmt.Sprintf("backup took %v", duration) + if !success { + title = "Backup failed" + msg = fmt.Sprintf("backup took %v, check the logs for errors", duration) + } + + if len(conf.Metrics.Heartbeat.URLs) > 0 || len(conf.Metrics.Heartbeat.FailureURLs) > 0 { + heartbeat.Send(conf.Metrics.Heartbeat, success) } if len(conf.Metrics.PushConfigs.Ntfy) > 0 { for _, pusher := range conf.Metrics.PushConfigs.Ntfy { pusher.ResolveToken() - err := ntfy.Notify(fmt.Sprintf("backup took %v", duration), *pusher) + err := ntfy.Notify(title, msg, *pusher) if err != nil { log.Warn().Str("push", "ntfy").Err(err).Msg("couldn't send message") } @@ -1161,7 +1171,7 @@ func runBackup(conf *types.Conf, num int) { if len(conf.Metrics.PushConfigs.Gotify) > 0 { for _, pusher := range conf.Metrics.PushConfigs.Gotify { pusher.ResolveToken() - err := gotify.Notify(fmt.Sprintf("backup took %v", duration), *pusher) + err := gotify.Notify(title, msg, *pusher) if err != nil { log.Warn().Str("push", "gotify").Err(err).Msg("couldn't send message") } @@ -1170,14 +1180,14 @@ func runBackup(conf *types.Conf, num int) { if len(conf.Metrics.PushConfigs.Apprise) > 0 { for _, pusher := range conf.Metrics.PushConfigs.Apprise { - err := apprise.Notify(fmt.Sprintf("backup took %v", duration), *pusher) + err := apprise.Notify(title, msg, *pusher) if err != nil { log.Warn().Str("push", "apprise").Err(err).Msg("couldn't send message") } } } - exitCode := logger.GetExitCode() - if exitCode != 0 { + + if !success { log.Warn().Msgf("Encountered at least one error during the run. Check the logs. Exiting with status=%d", exitCode) } diff --git a/metrics/apprise/apprise.go b/metrics/apprise/apprise.go index 042d0959..571ea46f 100644 --- a/metrics/apprise/apprise.go +++ b/metrics/apprise/apprise.go @@ -12,20 +12,22 @@ import ( ) type Request struct { - Body string `json:"body"` - Tags []string `json:"tags,omitempty"` - Urls []string `json:"urls,omitempty"` + Title string `json:"title,omitempty"` + Body string `json:"body"` + Tags []string `json:"tags,omitempty"` + Urls []string `json:"urls,omitempty"` } type ErrorMsg struct { Error string `json:"error"` } -func Notify(msg string, config types.AppriseConfig) error { +func Notify(title, msg string, config types.AppriseConfig) error { payload := Request{ - Body: msg, - Urls: config.Urls, - Tags: config.Tags, + Title: title, + Body: msg, + Urls: config.Urls, + Tags: config.Tags, } jsonData, err := json.Marshal(payload) diff --git a/metrics/apprise/apprise_test.go b/metrics/apprise/apprise_test.go index 002e7875..1726f938 100644 --- a/metrics/apprise/apprise_test.go +++ b/metrics/apprise/apprise_test.go @@ -42,7 +42,7 @@ func TestNotifySendsExpectedRequest(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.AppriseConfig{ + err := Notify("Backup done", "backup complete", types.AppriseConfig{ Url: server.URL, Config: "team", Tags: []string{"backup", "nightly"}, @@ -61,7 +61,7 @@ func TestNotifyReturnsAPIError(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.AppriseConfig{Url: server.URL}) + err := Notify("Backup done", "backup complete", types.AppriseConfig{Url: server.URL}) if err == nil { t.Fatal("expected apprise error") } diff --git a/metrics/gotify/gotify.go b/metrics/gotify/gotify.go index 07a0ae9f..6cbdbb82 100644 --- a/metrics/gotify/gotify.go +++ b/metrics/gotify/gotify.go @@ -11,7 +11,7 @@ import ( "github.com/cooperspencer/gickup/types" ) -func Notify(msg string, config types.PushConfig) error { +func Notify(title, msg string, config types.PushConfig) error { if !strings.HasSuffix(config.Url, "/") { config.Url += "/" } @@ -20,7 +20,7 @@ func Notify(msg string, config types.PushConfig) error { payload := map[string]string{} payload["message"] = msg - payload["title"] = "Backup done" + payload["title"] = title body, err := json.Marshal(payload) if err != nil { diff --git a/metrics/gotify/gotify_test.go b/metrics/gotify/gotify_test.go index 71ab9cf2..a08b3eca 100644 --- a/metrics/gotify/gotify_test.go +++ b/metrics/gotify/gotify_test.go @@ -38,7 +38,7 @@ func TestNotifySendsExpectedPayload(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) + err := Notify("Backup done", "backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) if err != nil { t.Fatalf("Notify() error = %v", err) } @@ -52,7 +52,7 @@ func TestNotifyReturnsStatusError(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) + err := Notify("Backup done", "backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) if err == nil { t.Fatal("expected status error") } diff --git a/metrics/heartbeat/heartbeat.go b/metrics/heartbeat/heartbeat.go index 7418bdad..2b6f7ba5 100644 --- a/metrics/heartbeat/heartbeat.go +++ b/metrics/heartbeat/heartbeat.go @@ -7,9 +7,21 @@ import ( "github.com/rs/zerolog/log" ) -func Send(conf types.HeartbeatConfig) { - for _, u := range conf.URLs { - log.Info().Str("url", u).Msg("sending heartbeat") +// Send pings the configured heartbeat URLs. +// +// On a successful backup run, conf.URLs is pinged. On a failed run, +// conf.FailureURLs is pinged instead, so monitoring services such as +// healthchecks.io that expose a dedicated "failure" endpoint can be +// notified immediately instead of only alerting once the regular +// heartbeat times out. +func Send(conf types.HeartbeatConfig, success bool) { + urls := conf.URLs + if !success { + urls = conf.FailureURLs + } + + for _, u := range urls { + log.Info().Str("url", u).Bool("success", success).Msg("sending heartbeat") resp, err := http.Get(u) //nolint:noctx if err != nil { log.Error().Str("monitoring", "heartbeat").Msg(err.Error()) diff --git a/metrics/heartbeat/heartbeat_test.go b/metrics/heartbeat/heartbeat_test.go index 2781d6c8..f6f05d50 100644 --- a/metrics/heartbeat/heartbeat_test.go +++ b/metrics/heartbeat/heartbeat_test.go @@ -20,9 +20,39 @@ func TestSendCallsEachHeartbeatURL(t *testing.T) { })) defer server.Close() - Send(types.HeartbeatConfig{URLs: []string{server.URL, server.URL}}) + Send(types.HeartbeatConfig{URLs: []string{server.URL, server.URL}}, true) if got := atomic.LoadInt32(&hits); got != 2 { t.Fatalf("hits = %d, want 2", got) } } + +func TestSendCallsFailureURLsOnFailure(t *testing.T) { + t.Parallel() + + var successHits, failureHits int32 + + successServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + atomic.AddInt32(&successHits, 1) + w.WriteHeader(http.StatusNoContent) + })) + defer successServer.Close() + + failureServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + atomic.AddInt32(&failureHits, 1) + w.WriteHeader(http.StatusNoContent) + })) + defer failureServer.Close() + + Send(types.HeartbeatConfig{ + URLs: []string{successServer.URL}, + FailureURLs: []string{failureServer.URL}, + }, false) + + if got := atomic.LoadInt32(&successHits); got != 0 { + t.Fatalf("successHits = %d, want 0", got) + } + if got := atomic.LoadInt32(&failureHits); got != 1 { + t.Fatalf("failureHits = %d, want 1", got) + } +} diff --git a/metrics/ntfy/ntfy.go b/metrics/ntfy/ntfy.go index 6c9437c1..7bb3c06d 100644 --- a/metrics/ntfy/ntfy.go +++ b/metrics/ntfy/ntfy.go @@ -10,7 +10,7 @@ import ( "github.com/cooperspencer/gickup/types" ) -func Notify(msg string, config types.PushConfig) error { +func Notify(title, msg string, config types.PushConfig) error { url := config.Url payload := strings.NewReader(msg) @@ -21,7 +21,7 @@ func Notify(msg string, config types.PushConfig) error { } req.Header.Add("Content-Type", "text/plain") - req.Header.Add("Title", "Backup done") + req.Header.Add("Title", title) if config.Email != "" { req.Header.Add("Email", config.Email) } diff --git a/metrics/ntfy/ntfy_test.go b/metrics/ntfy/ntfy_test.go index 145803c8..0d5b2875 100644 --- a/metrics/ntfy/ntfy_test.go +++ b/metrics/ntfy/ntfy_test.go @@ -47,7 +47,7 @@ func TestNotifyAddsEmailHeader(t *testing.T) { })) defer server.Close() - err := Notify("backup took 2m", types.PushConfig{ + err := Notify("Backup done", "backup took 2m", types.PushConfig{ Url: server.URL, Token: "my-token", Email: "user@example.com", @@ -78,7 +78,7 @@ func TestNotifyUsesBasicAuth(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.PushConfig{Url: server.URL, User: "andy", Password: "password"}) + err := Notify("Backup done", "backup complete", types.PushConfig{Url: server.URL, User: "andy", Password: "password"}) if err != nil { t.Fatalf("Notify() error = %v", err) } @@ -87,7 +87,7 @@ func TestNotifyUsesBasicAuth(t *testing.T) { func TestNotifyRequiresCredentials(t *testing.T) { t.Parallel() - err := Notify("backup complete", types.PushConfig{Url: "https://ntfy.sh/topic"}) + err := Notify("Backup done", "backup complete", types.PushConfig{Url: "https://ntfy.sh/topic"}) if err == nil { t.Fatal("expected credential error") } @@ -105,7 +105,7 @@ func TestNotifyReturnsStatusError(t *testing.T) { })) defer server.Close() - err := Notify("backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) + err := Notify("Backup done", "backup complete", types.PushConfig{Url: server.URL, Token: "secret-token"}) if err == nil { t.Fatal("expected status error") } diff --git a/types/types.go b/types/types.go index 86c3ddf6..64f7a1e1 100644 --- a/types/types.go +++ b/types/types.go @@ -90,7 +90,8 @@ type PrometheusConfig struct { // HeartbeatConfig TODO. type HeartbeatConfig struct { - URLs []string `yaml:"urls"` + URLs []string `yaml:"urls"` + FailureURLs []string `yaml:"failure_urls"` } // PushConfig TODO. From 109cc1eda4feed3b24f9abc14247876ad771428e Mon Sep 17 00:00:00 2001 From: Danny Sauer Date: Wed, 9 Sep 2026 11:34:36 -0500 Subject: [PATCH 33/44] fix(sourcehut): keep logging context local to each operation (#435) --- gitlab/gitlab_test.go | 4 ++++ sourcehut/sourcehut.go | 7 +------ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/gitlab/gitlab_test.go b/gitlab/gitlab_test.go index c05d5a95..22ac98e2 100644 --- a/gitlab/gitlab_test.go +++ b/gitlab/gitlab_test.go @@ -7,6 +7,8 @@ import ( ) func TestGetRepoVisibility(t *testing.T) { + t.Parallel() + tests := []struct { name string configured string @@ -23,6 +25,8 @@ func TestGetRepoVisibility(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { + t.Parallel() + if got := getRepoVisibility(test.configured, test.sourcePrivate); got != test.want { t.Fatalf("getRepoVisibility(%q, %t) = %q, want %q", test.configured, test.sourcePrivate, got, test.want) } diff --git a/sourcehut/sourcehut.go b/sourcehut/sourcehut.go index e40ce628..1ceb4caa 100644 --- a/sourcehut/sourcehut.go +++ b/sourcehut/sourcehut.go @@ -13,11 +13,8 @@ import ( "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" graphqlclient "github.com/hasura/go-graphql-client" - "github.com/rs/zerolog" ) -var sub zerolog.Logger - const defaultSourcehutURL = "https://git.sr.ht" func normalizeBearerToken(token string) string { @@ -250,7 +247,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { for _, repo := range conf.Source.Sourcehut { repo.URL = normalizeURL(repo.URL) - sub = logger.CreateSubLogger("stage", "sourcehut", "url", repo.URL) + sub := logger.CreateSubLogger("stage", "sourcehut", "url", repo.URL) err := repo.Filter.ParseDuration() if err != nil { sub.Warn(). @@ -374,8 +371,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { destination.URL = normalizeURL(destination.URL) - sub = logger.CreateSubLogger("stage", "sourcehut", "url", destination.URL) - token := destination.GetToken() endpoint := graphQLEndpoint(destination.URL) configuredUser := strings.TrimPrefix(destination.User, "~") From 55882ef63e5fc1dfe7b1c2f401a65342a23d99bc Mon Sep 17 00:00:00 2001 From: Danny Sauer Date: Wed, 9 Sep 2026 11:34:52 -0500 Subject: [PATCH 34/44] fix(github): use App installation token for destination pushes (#433) Pass the resolved destination credential through GetOrCreate to the push helper. Cover App and configured tokens on github.com and Enterprise Server, API failures, and the backup call chain. --- conf.example.yml | 6 +- github/github.go | 21 +++--- github/github_test.go | 146 +++++++++++++++++++++++++++++++++++++++ local/local.go | 4 +- local/local_push_test.go | 90 ++++++++++++++++++++++++ main.go | 14 ++-- main_github_test.go | 97 ++++++++++++++++++++++++++ 7 files changed, 357 insertions(+), 21 deletions(-) create mode 100644 local/local_push_test.go create mode 100644 main_github_test.go diff --git a/conf.example.yml b/conf.example.yml index 0866cd43..e63e7733 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -276,10 +276,12 @@ destination: repositories: private # private, public, default: private force: false # force push to destination # alternatively, authenticate with a GitHub App: + # requires Contents: write, plus Administration: write to create repositories # - app_id: 123456 # GitHub App ID (numeric) - # app_installation_id: 78901234 # Installation ID of the App + # app_installation_id: 78901234 # installation on the destination organization, including for enterprise-owned Apps # app_private_key_file: /path/to/private.pem # path to the App's RSA private key PEM file - # organization: whatever # org to back up into (leave empty for personal account) + # organization: whatever # destination organization; required for App authentication + # url: https://github.example.com # GitHub Enterprise Server base URL; omit for github.com # visibility: # repositories: private # force: false diff --git a/github/github.go b/github/github.go index c2e60935..cc412ad6 100644 --- a/github/github.go +++ b/github/github.go @@ -497,12 +497,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { return repos, ran } -// GetOrCreate Get or create a repository -func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { +// GetOrCreate returns the destination's clone URL and resolved push token, +// creating the repository if needed. +func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, string, error) { sub = logger.CreateSubLogger("stage", "github", "url", githubInstanceURL(destination.URL)) - client, _, err := newGithubClient(context.TODO(), destination) + client, token, err := newGithubClient(context.TODO(), destination) if err != nil { - return "", err + return "", "", err } dest := types.GithubDestination{} @@ -510,14 +511,14 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { if destination.Organization == "" { user, _, err := client.Users.Get(context.TODO(), "") if err != nil { - return "", err + return "", "", err } dest.User = user login = *user.Login } else { organization, _, err := client.Organizations.Get(context.TODO(), destination.Organization) if err != nil { - return "", err + return "", "", err } dest.Organization = organization login = *organization.Login @@ -526,12 +527,12 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { r, _, err := client.Repositories.Get(context.TODO(), login, repo.Name) if err != nil { if !strings.Contains(err.Error(), "404 Not Found") { - return "", err + return "", "", err } if destination.Organization == "" { r, _, err = client.Repositories.Create(context.TODO(), "", &github.Repository{Name: github.String(repo.Name), Private: github.Bool(destination.Visibility.Repositories == "private"), Visibility: github.String(destination.Visibility.Repositories), Owner: dest.User}) if err != nil { - return "", err + return "", "", err } } else { if destination.Visibility.Repositories == "" { @@ -539,12 +540,12 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { } r, _, err = client.Repositories.Create(context.TODO(), *dest.Organization.Login, &github.Repository{Name: github.String(repo.Name), Private: github.Bool(destination.Visibility.Repositories == "private"), Visibility: github.String(destination.Visibility.Repositories), Organization: dest.Organization}) if err != nil { - return "", err + return "", "", err } } } - return *r.CloneURL, nil + return *r.CloneURL, token, nil } // GetIssues get issues diff --git a/github/github_test.go b/github/github_test.go index 46ebc835..d08cfc14 100644 --- a/github/github_test.go +++ b/github/github_test.go @@ -2,9 +2,18 @@ package github import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "net/http" + "net/http/httptest" "os" "path/filepath" + "strings" "testing" + "time" "github.com/cooperspencer/gickup/types" ) @@ -82,3 +91,140 @@ func TestNewGithubClientAppAuthMissingKeyFile(t *testing.T) { t.Fatal("expected error when App private key file does not exist") } } + +//nolint:paralleltest // Replaces http.DefaultTransport and calls code with a package-level logger. +func TestGetOrCreateReturnsResolvedToken(t *testing.T) { + const wantToken = "installation-token-for-test" + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatalf("rsa.GenerateKey() error = %v", err) + } + keypath := filepath.Join(t.TempDir(), "app.pem") + pemBytes := pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }) + if err := os.WriteFile(keypath, pemBytes, 0o600); err != nil { + t.Fatalf("os.WriteFile() error = %v", err) + } + + tokenFile := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(tokenFile, []byte(wantToken+"\n"), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("GITHUB_DESTINATION_TEST_TOKEN", wantToken) + + for _, instance := range []struct { + name, url, apiURL, cloneURL string + }{ + {"github.com", "", "https://api.github.com", "https://github.com/example/repo.git"}, + {"enterprise-server", "https://github.example.com/", "https://github.example.com/api/v3", "https://github.example.com/example/repo.git"}, + } { + for _, tc := range []struct { + name, auth, failAt string + create bool + }{ + {name: "app-existing", auth: "app"}, + {name: "app-create", auth: "app", create: true}, + {name: "pat-existing", auth: "pat"}, + {name: "pat-create", auth: "pat", create: true}, + {name: "environment-token", auth: "environment"}, + {name: "token-file", auth: "file"}, + {name: "token-error", auth: "app", failAt: "/app/installations/4242/access_tokens"}, + {name: "organization-error", auth: "app", failAt: "/orgs/example"}, + {name: "repository-error", auth: "app", failAt: "/repos/example/repo"}, + {name: "creation-error", auth: "app", create: true, failAt: "/orgs/example/repos"}, + } { + t.Run(instance.name+"/"+tc.name, func(t *testing.T) { + destination := types.GenRepo{URL: instance.url, Organization: "example"} + switch tc.auth { + case "app": + destination.AppID = 99 + destination.AppInstallationID = 4242 + destination.AppPrivateKeyFile = keypath + destination.Token = "unused-configured-token" + case "pat": + destination.Token = wantToken + case "environment": + destination.Token = "GITHUB_DESTINATION_TEST_TOKEN" + case "file": + destination.TokenFile = tokenFile + } + + created, failed := false, false + // These fixtures rely on clients using http.DefaultTransport to intercept github.com requests. + oldTransport := http.DefaultTransport + t.Cleanup(func() { http.DefaultTransport = oldTransport }) + http.DefaultTransport = roundTripperFunc(func(r *http.Request) (*http.Response, error) { + w := httptest.NewRecorder() + w.Header().Set("Content-Type", "application/json") + if !strings.HasPrefix(r.URL.String(), instance.apiURL+"/") { + t.Errorf("unexpected API URL: %s", r.URL) + w.WriteHeader(http.StatusNotFound) + return w.Result(), nil + } + endpoint := strings.TrimPrefix(r.URL.String(), instance.apiURL) + if endpoint == "/app/installations/4242/access_tokens" { + if r.Method != http.MethodPost || !strings.HasPrefix(r.Header.Get("Authorization"), "Bearer ") { + t.Error("installation token request must use POST and App JWT authentication") + } + } else if !strings.HasSuffix(r.Header.Get("Authorization"), " "+wantToken) { + t.Error("API request did not use the resolved token") + } + if endpoint == tc.failAt { + failed = true + w.WriteHeader(http.StatusForbidden) + return w.Result(), nil + } + switch r.Method + " " + endpoint { + case "POST /app/installations/4242/access_tokens": + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]string{ + "token": wantToken, "expires_at": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), + }) + case "GET /orgs/example": + _, _ = w.WriteString(`{"login":"example"}`) + case "GET /repos/example/repo": + if tc.create { + w.WriteHeader(http.StatusNotFound) + _, _ = w.WriteString(`{"message":"Not Found"}`) + } else { + _ = json.NewEncoder(w).Encode(map[string]string{"clone_url": instance.cloneURL}) + } + case "POST /orgs/example/repos": + created = true + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]string{"clone_url": instance.cloneURL}) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusNotFound) + } + return w.Result(), nil + }) + + cloneURL, token, err := GetOrCreate(destination, types.Repo{Name: "repo"}) + if tc.failAt != "" { + if !failed || err == nil || cloneURL != "" || token != "" { + t.Fatalf("API error returned (%q, %q, %v), want empty URL and token with an error", cloneURL, token, err) + } + return + } + if err != nil { + t.Fatal(err) + } + if cloneURL != instance.cloneURL || token != wantToken { + t.Fatalf("GetOrCreate() = (%q, %q), want (%q, %q)", cloneURL, token, instance.cloneURL, wantToken) + } + if created != tc.create { + t.Fatalf("repository created = %v, want %v", created, tc.create) + } + }) + } + } +} + +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { + return f(r) +} diff --git a/local/local.go b/local/local.go index fb42f866..e4fc5c31 100644 --- a/local/local.go +++ b/local/local.go @@ -659,9 +659,9 @@ func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repositor return r, err } -func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url string, lfs bool) error { +// CreateRemotePush pushes to url using the caller's resolved token or SSH configuration. +func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url, token string, lfs bool) error { sub = logger.CreateSubLogger("stage", "tempclone", "url", url) - token := destination.GetToken() var auth transport.AuthMethod if destination.SSH { if destination.SSHKey == "" { diff --git a/local/local_push_test.go b/local/local_push_test.go new file mode 100644 index 00000000..7c4fa5f8 --- /dev/null +++ b/local/local_push_test.go @@ -0,0 +1,90 @@ +package local + +import ( + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/cooperspencer/gickup/types" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" + "github.com/go-git/go-git/v5/plumbing/transport" +) + +//nolint:paralleltest // CreateRemotePush assigns the package-level logger. +func TestCreateRemotePushUsesProvidedToken(t *testing.T) { + const want = "resolved-token-for-test" + + dir := t.TempDir() + repo, err := git.PlainInit(dir, false) + if err != nil { + t.Fatalf("git.PlainInit() error = %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte("x\n"), 0o644); err != nil { + t.Fatalf("os.WriteFile() error = %v", err) + } + wt, err := repo.Worktree() + if err != nil { + t.Fatalf("Worktree() error = %v", err) + } + if _, err := wt.Add("README.md"); err != nil { + t.Fatalf("Add() error = %v", err) + } + if _, err := wt.Commit("seed", &git.CommitOptions{ + Author: &object.Signature{Name: "t", Email: "t@example.com", When: time.Now()}, + }); err != nil { + t.Fatalf("Commit() error = %v", err) + } + + var passwordsMu sync.Mutex + var passwords []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, password, _ := r.BasicAuth() + passwordsMu.Lock() + passwords = append(passwords, password) + passwordsMu.Unlock() + w.Header().Set("WWW-Authenticate", `Basic realm="git"`) + w.WriteHeader(http.StatusUnauthorized) + })) + defer server.Close() + + for _, configuredToken := range []string{"", "unused-configured-token"} { + destination := types.GenRepo{Token: configuredToken} + err := CreateRemotePush(repo, destination, server.URL+"/org/repo.git", want, false) + if !errors.Is(err, transport.ErrAuthenticationRequired) { + t.Fatalf("push error = %v, want authentication required", err) + } + passwordsMu.Lock() + gotPasswords := passwords + passwords = nil + passwordsMu.Unlock() + if len(gotPasswords) == 0 { + t.Fatal("push did not reach the server") + } + for _, got := range gotPasswords { + if got != want { + t.Fatalf("push password = %q, want %q", got, want) + } + } + } + + reopened, err := git.PlainOpen(dir) + if err != nil { + t.Fatal(err) + } + remotes, err := reopened.Remotes() + if err != nil { + t.Fatal(err) + } + for _, remote := range remotes { + urls := remote.Config().URLs + if len(urls) != 1 || urls[0] != server.URL+"/org/repo.git" { + t.Fatalf("persisted remote URLs = %v, want original credential-free URL", urls) + } + } +} diff --git a/main.go b/main.go index f70168ba..ba46a42b 100644 --- a/main.go +++ b/main.go @@ -568,7 +568,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, d.GetToken(), r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -650,7 +650,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, d.GetToken(), r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -736,7 +736,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, d.GetToken(), r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -808,7 +808,7 @@ func backup(repos []types.Repo, conf *types.Conf) { } } - cloneurl, err := github.GetOrCreate(d, r) + cloneurl, destinationtoken, err := github.GetOrCreate(d, r) if err != nil { log.Error(). Str("stage", "github"). @@ -818,7 +818,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, destinationtoken, r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -896,7 +896,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, d.GetToken(), r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). @@ -975,7 +975,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + err = local.CreateRemotePush(temprepo, d, cloneurl, d.GetToken(), r.Origin.LFS) if err != nil { if errors.Is(err, git.NoErrAlreadyUpToDate) { log.Info(). diff --git a/main_github_test.go b/main_github_test.go new file mode 100644 index 00000000..3616f440 --- /dev/null +++ b/main_github_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/json" + "encoding/pem" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/cooperspencer/gickup/types" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" +) + +//nolint:paralleltest // backup calls providers that assign package-level loggers. +func TestBackupGitHubAppPushCredential(t *testing.T) { + const wantToken = "installation-token-for-test" + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + keyFile := filepath.Join(t.TempDir(), "app.pem") + if err := os.WriteFile(keyFile, pem.EncodeToMemory(&pem.Block{ + Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key), + }), 0o600); err != nil { + t.Fatal(err) + } + + sourceDir := t.TempDir() + source, err := git.PlainInit(sourceDir, false) + if err != nil { + t.Fatal(err) + } + worktree, err := source.Worktree() + if err != nil { + t.Fatal(err) + } + if _, err := worktree.Commit("seed", &git.CommitOptions{ + AllowEmptyCommits: true, + Author: &object.Signature{Name: "test", Email: "test@example.com", When: time.Now()}, + }); err != nil { + t.Fatal(err) + } + + var passwordsMu sync.Mutex + var passwords []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch r.Method + " " + r.URL.Path { + case "POST /api/v3/app/installations/42/access_tokens": + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]string{ + "token": wantToken, "expires_at": time.Now().Add(time.Hour).UTC().Format(time.RFC3339), + }) + case "GET /api/v3/orgs/example": + _, _ = w.Write([]byte(`{"login":"example"}`)) + case "GET /api/v3/repos/example/repo": + _ = json.NewEncoder(w).Encode(map[string]string{"clone_url": "http://" + r.Host + "/example/repo.git"}) + case "GET /example/repo.git/info/refs": + _, password, _ := r.BasicAuth() + passwordsMu.Lock() + passwords = append(passwords, password) + passwordsMu.Unlock() + w.WriteHeader(http.StatusUnauthorized) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + + backup([]types.Repo{{Name: "repo", URL: sourceDir}}, &types.Conf{ + Destination: types.Destination{Github: []types.GenRepo{{ + URL: server.URL, Organization: "example", + AppID: 1, AppInstallationID: 42, AppPrivateKeyFile: keyFile, + }}}, + }) + passwordsMu.Lock() + gotPasswords := passwords + passwords = nil + passwordsMu.Unlock() + if len(gotPasswords) == 0 { + t.Fatal("backup did not attempt a push") + } + for _, got := range gotPasswords { + if got != wantToken { + t.Fatalf("backup push password = %q, want resolved installation token", got) + } + } +} From 64e81c3b14a0c96b8adb5f45c107f6c4545c347c Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Fri, 11 Sep 2026 13:27:53 +0200 Subject: [PATCH 35/44] added lfs to s3 (#437) --- conf.example.yml | 1 + gickup_spec.json | 4 ++++ local/local.go | 6 ++++++ main.go | 1 + types/types.go | 1 + 5 files changed, 13 insertions(+) diff --git a/conf.example.yml b/conf.example.yml index e63e7733..d1053d57 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -331,6 +331,7 @@ destination: storageclass: "" # storage class for all repos to be uploaded to this S3 bucket. E.g. for AWS: STANDARD, STANDARD_IA, GLACIER, etc. src_repo_url_tag_key: "" # if set, tag each uploaded object with this key and the source repo URL as value. Leave unset to disable tagging. datecreatedir: false # if true, gickup will create a directory for backup with the creation date + lfs: true # clone lfs repos, !! ATTENTION !! git and git-lfs must be installed on the system! azureblob: - url: https://yourstorageaccount.blob.core.windows.net # blob storage endpoint for your storage account, can be a SAS url container: name of your blob container diff --git a/gickup_spec.json b/gickup_spec.json index ab9600fe..d024dbe1 100644 --- a/gickup_spec.json +++ b/gickup_spec.json @@ -618,6 +618,10 @@ "datecreatedir": { "type": "boolean", "description": "If true, gickup will create a directory for backup with the creation date" + }, + "lfs": { + "type": "boolean", + "description": "uses lfs to clone repositories" } }, "additionalProperties": false diff --git a/local/local.go b/local/local.go index e4fc5c31..b9220564 100644 --- a/local/local.go +++ b/local/local.go @@ -604,6 +604,12 @@ func tempCloneBase(repo types.Repo, tempdir string, isBare bool) (*git.Repositor return r, lfsErr } + // Bare backups already contain all refs and LFS objects. + // Checkout and pull require a working tree and must not run for these clones. + if isBare { + return r, nil + } + // Get the symbolic reference for HEAD headRef, err := r.Head() if err != nil { diff --git a/main.go b/main.go index ba46a42b..c5dce47f 100644 --- a/main.go +++ b/main.go @@ -256,6 +256,7 @@ func backup(repos []types.Repo, conf *types.Conf) { } defer os.RemoveAll(tempdir) + r.Origin.LFS = d.LFS tempClonePath := path.Join(tempdir, r.Name) _, err = local.TempCloneBare(r, tempClonePath) if err != nil { diff --git a/types/types.go b/types/types.go index 64f7a1e1..92838c5f 100644 --- a/types/types.go +++ b/types/types.go @@ -595,6 +595,7 @@ type S3Repo struct { StorageClass string `yaml:"storageclass"` DateCreateDir bool `yaml:"datecreatedir"` SrcRepoUrlTagKey *string `yaml:"src_repo_url_tag_key"` + LFS bool `yaml:"lfs"` } func (s3 S3Repo) GetKey(accessString string) (string, error) { From 3697a140ec810e5927078276c09d5f3a5e724195 Mon Sep 17 00:00:00 2001 From: Ryan Date: Sun, 13 Sep 2026 14:32:59 +0800 Subject: [PATCH 36/44] Add Winget to distribution packages in README (#440) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 40add751..ff27402e 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ https://github.com/cooperspencer/gickup/discussions |Homebrew|[gickup](https://formulae.brew.sh/formula/gickup#default)|| |Fedora|[gickup](https://copr.fedorainfracloud.org/coprs/frostyx/gickup/)|[FrostyX](https://github.com/FrostyX)| |Scoop|[gickup](https://scoop.sh/#/apps?q=gickup&id=493e856707843828b4491004edfb6ddedc4bd1e1)|| +|Winget|[gickup](https://github.com/microsoft/winget-pkgs/tree/master/manifests/c/cooperspencer/gickup)|| ## Issues The mirroring to Gitlab doesn't work, or at least I can't test it properly because I have no access to a Gitlab EE instance. From ef37e12c6c574f1a407e1fc11020e766b425f641 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Sun, 13 Sep 2026 08:33:56 +0200 Subject: [PATCH 37/44] added test and fixed the "-C" parameter (#441) --- gitcmd/gitcmd.go | 2 +- gitcmd/gitcmd_test.go | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/gitcmd/gitcmd.go b/gitcmd/gitcmd.go index f8b3aec6..b5a67574 100644 --- a/gitcmd/gitcmd.go +++ b/gitcmd/gitcmd.go @@ -137,7 +137,7 @@ func (g GitCmd) Checkout(path, branch string) error { if err != nil { return err } - args := []string{"checkout", branch} + args := []string{"-C", path, "checkout", branch} cmd := g.Command(context.Background(), nil, args...) output, err := cmd.CombinedOutput() diff --git a/gitcmd/gitcmd_test.go b/gitcmd/gitcmd_test.go index 1ada2a44..7f40a881 100644 --- a/gitcmd/gitcmd_test.go +++ b/gitcmd/gitcmd_test.go @@ -265,6 +265,22 @@ func TestGitCmd_LFSFetchCommand(t *testing.T) { } } +func TestGitCmd_CheckoutCommand(t *testing.T) { + t.Parallel() + + g := GitCmd{CMD: "git"} + + // Checkout must scope the command to the repo path with -C, otherwise + // git runs against the process's cwd instead of the cloned repo. + checkoutArgs := []string{"-C", "/tmp/repo", "checkout", "main"} + cmd := g.Command(context.Background(), nil, checkoutArgs...) + wantArgs := []string{"git", "-C", "/tmp/repo", "checkout", "main"} + + if !reflect.DeepEqual(cmd.Args, wantArgs) { + t.Errorf("cmd.Args = %v, want %v", cmd.Args, wantArgs) + } +} + func TestGitCmd_LocalCommandsHaveNoAuth(t *testing.T) { t.Parallel() From 611f638070b2e06eeaa01d9b89f4b4b8d1ba4ecd Mon Sep 17 00:00:00 2001 From: KismetDev <131729061+0xKismetDev@users.noreply.github.com> Date: Wed, 16 Sep 2026 07:53:56 +0200 Subject: [PATCH 38/44] Pass git credentials through a credential helper instead of http.extraHeader (#443) git lfs adds http.extraHeader to object downloads, which already carry an Authorization header from the LFS batch response. gitlab.com rejects the duplicate header with 400, so no LFS objects were downloaded since #425. --- gitcmd/gitcmd.go | 22 ++++++++++---- gitcmd/gitcmd_test.go | 68 +++++++++++++++++++------------------------ local/local_test.go | 21 +++++++++---- 3 files changed, 62 insertions(+), 49 deletions(-) diff --git a/gitcmd/gitcmd.go b/gitcmd/gitcmd.go index b5a67574..7283bb30 100644 --- a/gitcmd/gitcmd.go +++ b/gitcmd/gitcmd.go @@ -2,7 +2,6 @@ package gitcmd import ( "context" - "encoding/base64" "errors" "fmt" "os" @@ -15,15 +14,28 @@ type Auth struct { Password string } +// envAuthHelper answers git's credential "get" request from the environment, +// so credentials never end up in URLs, command arguments or git config files. +const envAuthHelper = `!f() { test "$1" = get || return 0; echo "username=${GICKUP_GIT_USERNAME}"; echo "password=${GICKUP_GIT_PASSWORD}"; }; f` + +// Env passes the credentials through a credential helper instead of +// http.extraHeader. git lfs adds extra headers to object downloads as well, +// where the LFS server already provides its own Authorization header, and +// servers like gitlab.com reject the duplicate header with 400 Bad Request. func (a *Auth) Env() []string { if a == nil || (a.Username == "" && a.Password == "") { return nil } - encoded := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s:%s", a.Username, a.Password))) return []string{ - "GIT_CONFIG_COUNT=1", - "GIT_CONFIG_KEY_0=http.extraHeader", - fmt.Sprintf("GIT_CONFIG_VALUE_0=Authorization: Basic %s", encoded), + "GICKUP_GIT_USERNAME=" + a.Username, + "GICKUP_GIT_PASSWORD=" + a.Password, + "GIT_CONFIG_COUNT=2", + // The empty value resets helpers from system or global config, + // so no other helper stores these credentials. + "GIT_CONFIG_KEY_0=credential.helper", + "GIT_CONFIG_VALUE_0=", + "GIT_CONFIG_KEY_1=credential.helper", + "GIT_CONFIG_VALUE_1=" + envAuthHelper, } } diff --git a/gitcmd/gitcmd_test.go b/gitcmd/gitcmd_test.go index 7f40a881..75f86ec0 100644 --- a/gitcmd/gitcmd_test.go +++ b/gitcmd/gitcmd_test.go @@ -2,13 +2,23 @@ package gitcmd import ( "context" - "encoding/base64" - "fmt" "reflect" "strings" "testing" ) +func expectedAuthEnv(username, password string) []string { + return []string{ + "GICKUP_GIT_USERNAME=" + username, + "GICKUP_GIT_PASSWORD=" + password, + "GIT_CONFIG_COUNT=2", + "GIT_CONFIG_KEY_0=credential.helper", + "GIT_CONFIG_VALUE_0=", + "GIT_CONFIG_KEY_1=credential.helper", + "GIT_CONFIG_VALUE_1=" + envAuthHelper, + } +} + func TestAuth_Env_Nil(t *testing.T) { t.Parallel() @@ -36,12 +46,7 @@ func TestAuth_Env_GitHubApp(t *testing.T) { } env := auth.Env() - expectedHeader := "Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte("x-access-token:test-token")) - expectedEnv := []string{ - "GIT_CONFIG_COUNT=1", - "GIT_CONFIG_KEY_0=http.extraHeader", - "GIT_CONFIG_VALUE_0=" + expectedHeader, - } + expectedEnv := expectedAuthEnv("x-access-token", "test-token") if !reflect.DeepEqual(env, expectedEnv) { t.Fatalf("auth.Env() = %v, want %v", env, expectedEnv) @@ -57,12 +62,7 @@ func TestAuth_Env_UserPassword(t *testing.T) { } env := auth.Env() - expectedHeader := "Authorization: Basic " + base64.StdEncoding.EncodeToString([]byte("octocat:secretpassword456")) - expectedEnv := []string{ - "GIT_CONFIG_COUNT=1", - "GIT_CONFIG_KEY_0=http.extraHeader", - "GIT_CONFIG_VALUE_0=" + expectedHeader, - } + expectedEnv := expectedAuthEnv("octocat", "secretpassword456") if !reflect.DeepEqual(env, expectedEnv) { t.Fatalf("auth.Env() = %v, want %v", env, expectedEnv) @@ -97,27 +97,14 @@ func TestGitCmd_Command_WithAuth(t *testing.T) { t.Fatal("cmd.Env is nil, want environment variables with auth config") } - hasConfigCount := false - hasConfigKey := false - hasConfigValue := false - - expectedValue := fmt.Sprintf("GIT_CONFIG_VALUE_0=Authorization: Basic %s", - base64.StdEncoding.EncodeToString([]byte("x-access-token:test-token"))) - + present := make(map[string]bool, len(cmd.Env)) for _, e := range cmd.Env { - if e == "GIT_CONFIG_COUNT=1" { - hasConfigCount = true - } - if e == "GIT_CONFIG_KEY_0=http.extraHeader" { - hasConfigKey = true - } - if e == expectedValue { - hasConfigValue = true - } + present[e] = true } - - if !hasConfigCount || !hasConfigKey || !hasConfigValue { - t.Errorf("cmd.Env missing required GIT_CONFIG variables, env = %v", cmd.Env) + for _, want := range expectedAuthEnv("x-access-token", "test-token") { + if !present[want] { + t.Errorf("cmd.Env missing %q, env = %v", want, cmd.Env) + } } // Verify the URL in cmd.Args does NOT contain token @@ -254,14 +241,19 @@ func TestGitCmd_LFSFetchCommand(t *testing.T) { t.Fatal("cmd.Env is nil, want auth environment for LFS fetch") } - foundExtraHeader := false + foundHelper := false for _, e := range cmd.Env { - if e == "GIT_CONFIG_KEY_0=http.extraHeader" { - foundExtraHeader = true + if e == "GIT_CONFIG_VALUE_1="+envAuthHelper { + foundHelper = true + } + // git lfs would add an extra header to object downloads next to the + // Authorization header from the LFS batch response. + if strings.Contains(e, "http.extraHeader") { + t.Errorf("cmd.Env sets http.extraHeader, which breaks LFS object downloads: %q", e) } } - if !foundExtraHeader { - t.Error("cmd.Env does not contain GIT_CONFIG_KEY_0=http.extraHeader for LFS fetch") + if !foundHelper { + t.Error("cmd.Env does not contain the credential helper for LFS fetch") } } diff --git a/local/local_test.go b/local/local_test.go index 66831655..79569b59 100644 --- a/local/local_test.go +++ b/local/local_test.go @@ -215,14 +215,23 @@ func TestToGitCmdAuth_FromTokenAuth_GitHubApp(t *testing.T) { } env := gitAuth.Env() - if len(env) != 3 { - t.Fatalf("expected 3 env vars, got %d: %v", len(env), env) + want := map[string]bool{ + "GICKUP_GIT_USERNAME=x-access-token": false, + "GICKUP_GIT_PASSWORD=ghs_testtoken123": false, + "GIT_CONFIG_KEY_1=credential.helper": false, } - if env[0] != "GIT_CONFIG_COUNT=1" || env[1] != "GIT_CONFIG_KEY_0=http.extraHeader" { - t.Errorf("unexpected env vars: %v", env) + for _, e := range env { + if _, ok := want[e]; ok { + want[e] = true + } + if strings.Contains(e, "http.extraHeader") { + t.Errorf("unexpected http.extraHeader in env: %s", e) + } } - if !strings.HasPrefix(env[2], "GIT_CONFIG_VALUE_0=Authorization: Basic ") { - t.Errorf("unexpected GIT_CONFIG_VALUE_0: %s", env[2]) + for e, found := range want { + if !found { + t.Errorf("env missing %q: %v", e, env) + } } } From 658eedec6e55b9b5a408030588ea6283de33abad Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Sat, 2 May 2026 21:11:22 +0200 Subject: [PATCH 39/44] added webinterface --- conf.example.yml | 3 + gickup_spec.json | 11 + main.go | 168 +++++++- types/types.go | 6 + webui/ui/index.html | 976 ++++++++++++++++++++++++++++++++++++++++++++ webui/ui/logo.png | Bin 0 -> 35084 bytes webui/webui.go | 282 +++++++++++++ 7 files changed, 1444 insertions(+), 2 deletions(-) create mode 100644 webui/ui/index.html create mode 100644 webui/ui/logo.png create mode 100644 webui/webui.go diff --git a/conf.example.yml b/conf.example.yml index d1053d57..0f8ecc9e 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -350,6 +350,9 @@ destination: structured: true # checks repos out like hostersite/user|organization/repo zip: false # if true, will zip the entire git repo into a single zip file and upload that instead datecreatedir: false # if true, gickup will create a directory for backup with the creation date +webui: # optional - enables the built-in web interface + addr: ":8282" # address and port to listen on + cron: 0 22 * * * # optional - when cron is not provided, the program runs once and exits. # Otherwise, it runs according to the cron schedule. # See timezone commentary in docker-compose.yml for making sure this container runs diff --git a/gickup_spec.json b/gickup_spec.json index d024dbe1..59b42ce3 100644 --- a/gickup_spec.json +++ b/gickup_spec.json @@ -15,6 +15,17 @@ "metrics": { "$ref": "#/definitions/metrics" }, + "webui": { + "type": "object", + "description": "Configuration for the built-in web interface", + "additionalProperties": false, + "properties": { + "addr": { + "type": "string", + "description": "Address and port to listen on, e.g. ':8282'" + } + } + }, "source": { "type": "object", "description": "Configuration for all git hoster source endpoints", diff --git a/main.go b/main.go index c5dce47f..95f6e630 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "reflect" "strconv" "strings" + "sync" "time" "github.com/alecthomas/kong" @@ -35,6 +36,7 @@ import ( "github.com/cooperspencer/gickup/sourcehut" "github.com/cooperspencer/gickup/types" "github.com/cooperspencer/gickup/webdav" + "github.com/cooperspencer/gickup/webui" "github.com/cooperspencer/gickup/whatever" "github.com/cooperspencer/gickup/zip" "github.com/go-git/go-git/v5" @@ -221,6 +223,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "local", d.Path).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("local").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "local", + DestAddr: d.Path, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } for _, d := range conf.Destination.S3 { @@ -335,6 +348,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "s3", d.Endpoint).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("s3").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "s3", + DestAddr: d.Endpoint, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -426,6 +450,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "azureblob", d.Container).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("azureblob").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "azureblob", + DestAddr: d.Container, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -599,6 +634,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "gitea", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -681,6 +727,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gogs").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "gogs", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -767,6 +824,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitlab", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("gitlab").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "gitlab", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } @@ -842,6 +910,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "github", "https://github.com").Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("github").Inc() + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "github", + DestAddr: "https://github.com", + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } } @@ -920,6 +999,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "onedev", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("onedev").Inc() os.RemoveAll(tempdir) + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "onedev", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } } @@ -999,6 +1089,17 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "sourcehut", d.URL).Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("sourcehut").Inc() os.RemoveAll(tempdir) + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, + RepoName: r.Name, + RepoURL: r.URL, + Owner: r.Owner, + Hoster: r.Hoster, + DestType: "sourcehut", + DestAddr: d.URL, + Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) } } } @@ -1219,8 +1320,10 @@ func playsForever(c *cron.Cron, conffiles []string, confs []*types.Conf) bool { if !cmp.Equal(confs, checkconfigs) { log.Info().Msg("config changed") log.Debug().Msg(cmp.Diff(confs, checkconfigs)) - for _, entry := range c.Entries() { - c.Remove(entry.ID) + if c != nil { + for _, entry := range c.Entries() { + c.Remove(entry.ID) + } } return true } @@ -1269,10 +1372,15 @@ func main() { Msgf("this is a %s", types.Blue("dry run")) } + var webuiOnce sync.Once + var webuiAddr string + _ = webuiAddr // will be set from config + init := true for { reload := false confs := []*types.Conf{} + confNames := []string{} for i, f := range cli.Configfiles { log.Info().Str("file", f). Msgf("Reading %s", types.Green(f)) @@ -1281,7 +1389,17 @@ func main() { log.Panic().Err(err).Msgf("there is an issue with %s", f) } cli.Configfiles[i] = absf + base := filepath.Base(absf) + before := len(confs) confs = append(confs, readConfigFile(absf)...) + added := len(confs) - before + for j := 0; j < added; j++ { + if added > 1 { + confNames = append(confNames, fmt.Sprintf("%s #%d", base, j+1)) + } else { + confNames = append(confNames, base) + } + } } logConf := confs[0].Log @@ -1292,6 +1410,46 @@ func main() { log.Logger = logger.CreateLogger(logConf) + // Determine webUI address from config file. + if webuiAddr == "" && confs[0].WebUI.Addr != "" { + webuiAddr = confs[0].WebUI.Addr + } + + // Register config summaries and run callback; start server once. + if webuiAddr != "" { + infos := make([]webui.ConfigInfo, len(confs)) + for i, c := range confs { + info := webui.ConfigInfo{ + Index: i, + Name: confNames[i], + Sources: c.Source.Count(), + Dests: c.Destination.Count(), + } + if c.HasValidCronSpec() { + info.CronSpec = c.Cron + if next, err := c.GetNextRun(); err == nil { + info.NextRun = next.Format(time.RFC3339) + } + } + infos[i] = info + } + webui.Global.SetConfigFiles(cli.Configfiles) + webui.Global.SetConfigs(infos) + currentConfs := confs + webui.Global.SetRunFunc(func(idx int) { + if idx < 0 { + for i, c := range currentConfs { + runBackup(c, i) + } + } else if idx < len(currentConfs) { + runBackup(currentConfs[idx], idx) + } + }) + webuiOnce.Do(func() { + go webui.Serve(webuiAddr) + }) + } + validcron := confs[0].HasValidCronSpec() var c *cron.Cron @@ -1350,6 +1508,12 @@ func main() { } reload = playsForever(c, cli.Configfiles, confs) log.Info().Msg("reloading config...") + } else if webuiAddr != "" { + // No cron but web UI is active: stay alive and watch for config changes. + reload = playsForever(nil, cli.Configfiles, confs) + if reload { + log.Info().Msg("reloading config...") + } } if !reload { break diff --git a/types/types.go b/types/types.go index 92838c5f..1ef94437 100644 --- a/types/types.go +++ b/types/types.go @@ -73,6 +73,11 @@ type Local struct { LFS bool `yaml:"lfs"` } +// WebUIConfig holds web interface settings. +type WebUIConfig struct { + Addr string `yaml:"addr"` +} + // Conf TODO. type Conf struct { Source Source `yaml:"source"` @@ -80,6 +85,7 @@ type Conf struct { Cron string `yaml:"cron"` Log Logging `yaml:"log"` Metrics Metrics `yaml:"metrics"` + WebUI WebUIConfig `yaml:"webui"` } // PrometheusConfig TODO. diff --git a/webui/ui/index.html b/webui/ui/index.html new file mode 100644 index 00000000..3f8bf226 --- /dev/null +++ b/webui/ui/index.html @@ -0,0 +1,976 @@ + + + + + + Gickup + + + + + + + +
+ + +
+
+
Total Backups
—
+
Successful
—
+
Failed
—
+
Unique Repos
—
+
Sources
—
+
+ +
+
+

Run Backup

+
+ +
+
+
+ + +
+ +
+ +
+
+

Backup History

+
+ + + +
+
+ +
+ + + + +
+ +
+
+
📦
+
No backup data yet
+
Backup entries will appear here once a run completes.
+
+
+
+
+ + +
+
+
+

Configuration

+
+ +
+
+
+
+ +
+ + +
+
+
+
+ + +
+
+

API Reference

+
+ +
+
GET/api/status
+
Return all recorded backup entries as a JSON array.
+
+
+
DELETE/api/status
+
Clear all recorded backup entries.
+
+ +
+
GET/api/configs
+
Return the list of loaded config blocks (name, sources, destinations, cron spec, next run).
+
+ +
+
GET/api/running
+
Returns {"running": true|false} — whether a backup run is currently in progress.
+
+ +
+
POST/api/run
+
Trigger an immediate backup run.
+ Body: {"index": -1} — -1 runs all configs, ≥0 runs the config at that index.
+ Returns 202 Accepted or 409 Conflict if already running. +
+
+ +
+
GET/api/config
+
Read a config file.
+ ?file=0 — zero-based file index (default 0).
+ ?list=1 — list all registered config files as JSON instead. +
+
+
+
POST/api/config
+
Overwrite a config file with the plain-text request body.
+ ?file=0 — zero-based file index (default 0). +
+
+ +
+
+
+ +
+ + + + diff --git a/webui/ui/logo.png b/webui/ui/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..1b0d3e820e1b1d03c6dd14bb10a8531c0cc89df8 GIT binary patch literal 35084 zcmeFYbx@qmwl_*}3j~5Y1Pku&5>|kzX3#0;jI0C7F?pEe-aPISmsd|YAwf3RQypfmzw5W!m z(zq+1QY1sOu=nN(9qu<~#Xfq-PaHx@4D!Fqtn~SPdARX=xc7OAq5PJfS>jVY_i3$g zRA^|eN8U($-)FeEHSQQ{kOe(+SoUl6Q*&2xPb~L~y_$PkxWB!}YRx1lzbGV+Ilnc2 zz6$y%eoxtUumhrs;AU$6360&dkke=t>(3%DgQnC9-a(aSpDt~)j$42$#}Q$yy9V;N zwWsXQkc_<)XR^0FbV_UL8z6XGzq9@|*)2}7t!AKai|=w$o$G^>r+CJ<>pIkf&Axl6 zgV@a#_wIX^oTt|QHrLaI((M^OU%yKw!cxDbB%{dZ-J`B3@dv7XO1FKx0acXgyTj-~ z2XRZ@A%{dk--q8e>#a8CIHi1_#>B?FZ4%lJLtpYoeZIDdSsu+gYiw(aUw$!<{i*^y zv30$Xf)UoL3Yj|_c?4l3a%38yAFVbku zQu2)xUS7`-5vT<6v8H4IThFllvYz+4+RpA5Np+!uF~>D)&4UGiH*n$Au1e z1UC!Eg^#-o4~W&wX|gFO82VOzhbA^p0jwVPEVhLrek10-jXtgt5dyW&TpxENtUvP9 zrF*^=wb&D|99tWX`c=6r@qx8hA(3ffaB_&z$+mU!yvc5SLb3Vfz&PqR`iv!xAfD0> zNa8J69MxLYCDxOW_3`3Tg;Etgn-(NJ2wRKm;g4&vy91m4B565NpJ>j$R3N%pN!5s0 zUpnZ=84FhR?2O5Mk8qRioXfs_({aVNA<9j<=SB4-RN$P2nI5@Zx5~b&fE5}i&1z`S z;&Ht?IfG=QW8dPraOk$^F>?*^X}oJ%ZX2t4*!|=ojK6SsY_R^j@Y+(#V;br;z`2GsTpz3qsx$A)qdR~D-vY6O)MJCsWqLn+HtY_6j%THoIc7rR*2hnUM zF`3n!&{|#c(ZsW7M`)WopxcHYuZkq+rRPg5FhGWPDqU^|)AuD!BO%=%ix11d`@~dc z<@V^dQ7KB!y9S4Io}-3USCwPok#HRi&j`GO7QQVPZ(NqE-C9aP*3dF&x85#OxV5%0 zTo7usIZi5O<7Ym*)2~j0R-iI+#H5egFE8YgDAR}8w_++B*VyZJYx3(5F@0qB5iGc| zu0a{03=uR4s~{WFwCSAC$(R!g9kOvF_Qnd82we!2y9!tVQ+5p@@v;nI)74{4@03;* zBk+s!ccDpp8O!oR zpW#tZwUvPEiJX_eTAkQozz^k37iN_@($vq*Yzc8+tM9S$(g#?(xN82aH^y#7yiOd6 zO)wzT7Ku%QsB}|s_F|15I5bT8yd|wus_GrA93P*Y$Yi@t5RKdFN+~-4cCP%k)Zbh0 zmwroX2seB*Fcb-w4fyyj?>RkAj`DVAtE+g)x`4&$UO(Mcef{;$4Cg_uET`vyf5n7~ zcNj-!O%D-67($t~X)Qa|ScP2|JbCVHP(atm!{5`Fa-|zRpPF`BcBP1ZXi|&fzumVO ze6LbTaYL=>bitM;rz>c4Rkq@F`%T(VT1)*Ma!mZCt4S^Cy>iAJmt6F0(;EeD#}A|C z{$H4;g_F^G9rFZTzvR8Bf2S;jS~ZP4>7$p@BJ^~vL80bOTw@Ts>H9^x+hbTb!-umg zsWH&DgUgmxVq%N+9xg<0KHp1g?PAbT%UkB1-0>^HwYvROKDw6(dSnC4BR<|)3(ZTd zMcP<@L>CL={oR$S0R&pm@Hd3Ir0N?lh7frrR7Za z<8(3D@z8zE7N3R)1MgLleO?8U9J&jFXu?)%2Z8@+0xP5OSSgC6kjeWq4L?<{cg{GeLVM4W%X#47Ol$&<{Z@!3nc!yF3 zSNqTCwrY9Z&eRraZ-g+*Wm|Z@XG*UJ2aO@j9Wp8%b)@yMXG{4LXv9(L)+0&YXIxZ*z*F-rG5S= zl#g9R)tZf>Eb?=^?A6RN-mC+3w?odgJ|@1yErIVxOU4N1yv8qQ|dfH>ej$8#X z6stG>R=6q+>4egHkx-2r?_pGBDuFMqB+Z+&85BxzC~|71qOAd~SBse((kn~kR&bmn zn#tclKr;a)-BfLUEGv4RI}#JYv5t`9PDHF=2zo30>o16B%QDa^&$KX^g}|d#5=WlL zPW(6ikv#3hgKw5I!_I8jJ}n5eon^{}Z&lGY1FVuw z2WM)f_sRFvitvdpRN&hMRWZCM&6z8FgQ}=Gh){lXnr)LlWm*0do+E5)e(VnKP^s~- z0SgeKl}im9^wahsMA&KMX}po|!nBp!JQ@D=`uvL%r4&*TVkCEp@m59%!LT1I8j2u} z)mGnq0T@aRiopP=y{>o8V;kGYT$m83Qxj?yEc?c`L2D3*pRH0up83ja_oO;=yTx`R zM&ny7F}qAB$yp6&jRM1BQv#J!5n@J>|1nl=N2XmmS%NbY`5|7~>%Cscd3Q!Sv~x0l zJH0P`*3(*a_XU?0HaaH!z1KG`Z%K4QL3vK>Sk%ZFdn*$W&z~jjF$XU<3Wg7)ZnDoM zXn6x@K6ivldSiBA(|A4c<;hl`*O$T5C-)ZSey#^ZvJX?Ju;Frdx%Ry8h|A%NuT+o7 zMt=YZL4n@Z)2to}KRZ?RRK+-1dHwMDgpfQD(sLWGjIIqNvfe|j%spk#N6D9TR zYBA-v;t2h%a6qVpw44sVHg|b)x`{oHiyY;MzKd$Rih6ZK#QG}P zSmuoD31zG^pB11^wYZM}r1j-ELF)_CeVwwsib+Ev6vKo;85Du^d4nLkL;$4G(fXRZ ze}Iz`Yq!+@5~EK$Du=9GGWZ<&{j(NlCsG*sx*$TQb-FYO1Eh>Irk<8SU^T6t$ee%; zLxLpb^}BZgZf1D6y%D*pRT^btIN)TR+3A3Vesd_<;$- zy|^76*=h1O%Hp+WJa=E=1P-zyoRPZ_E84#&@~hE~D_kR%bFDbZ`VirF-dnvNM!m(5 za20U1XTczs7@=8oid4j%gdj;*kp#a(8bISa6;VSFrf?$vVd9@nk=tdhdQlpy=8lPJ z^Ul%ugRnYA*=a4RhOU7Kn}=U7W=;nJ_s{^&9pSW2zVSP^#k#e&b!y22mk^&o@WAaV z);hBCjog+CSu|)4tCWR+`c>O@Pzu!4z&p`H8Z zE^!@+E3xi*%kA_{paIS?OWHYIpLG1y2m~r~SMtP58)t||gUWXUybjf?k z43hG*kVumr-jY}aL@W=V^Jg9-u&CeHNMg`%`1@h9=kz+=bfD>su$RaqTe8gEcPE$) zhZ8nGP`@!2TFiS)-2WVcDr=b2{!=iR_p~3Mx#i7%4s8r#72kALv8}6PhSO|4)zljN zOgzFHTD5oyRIJZbohMb!J)o(%w^#uJs|_>)R4tO#Zj$9_H;d^z*cIzmh!Mej9W`H~ zPZP(|LMkC4H5{GZL~hme+>>*wKe?QK%xeC{;vAU?CGz0OO+!g9-VCMQpc5Qg18Hd%)7_WuC{7MVLD)Ul(SDuV9a>^_s z6bUD5`cX1MV2k-ki4KzoV{Z=1($tVGF_lf_4}*i8d5{C@0RvrF1Yz+Tum__$!u*8-aA+U&Z9eq=*np zja9k&!^JD`&koZ5buI>(BvNKJB!mQBQC*ACg-+2aS#vb~S4u%-i2oVQj`;2Q&x3>!mjhM0N4*D-FrIt%(T_L1In&s;>%?fc6? zso@mTQF!18BcA56>Zp@~i={{S%CK&zga~B9iilY)3K#NP2lpzIBf$UMcoBn{liAWC z29LO+ygMYEE3+wtQOE+ZlU&omX*WCRq{#Xvl2pbA749AGgZ@WTXdC*#xdlE-l760N zLv{VMvQBuIB_joscC*0)C8~6wO(2{8Mcw8{GmMdV&6kSfpo3{qE=Q5@@}oJk2wegMjPc;jw4g6b*sr zs54Uurf+D_6(`C{+A*{(*MozC`XI^Hy?)MliFFfhXPJR)|?typcNm}QCCyVIaXeAUgmsx-P zfCKa9;?2>~m!{!)?Tc%U0-$h-&v{}4GY2$m+BrdH4+!hV@Et1pAq|MdqWy`IjqU=} zW3S)yud#2qMC=nKk6gYY-|d^3l498x;Vkl})XxD~Xha?b(Qlz1VvLsK`M-{X8*)YY z+A#%Tfb^JdIL$B$tuXNP8D!~2e6p23FAX;Zv)T_TFEXK6O2#$wl4@dHo2F47xDdg| zEC@TVS^MU%lWR~nhfU@5ECl!hts>Mu5D_LbX<)zJH32JxRk|q&$7;7_WJ9{5h7-NW`pqK8y#G%3OPZf>!(2pjFW@IKFL1g?+A40< zB}_iVN)$TbTecb!mR+Y9b;cyD74f}8vZn}oAv?<1`(0P8Z*lo&c@b+jfy0Txs7`=@ z6XGd6u9azHgcNh)D@!0-s;=dh`O_@dY`OH%jRKdBzS#FB863@r;K0TDd`}39&5Mh>n8gHK!+EgFa^q3 zfgEiC9%|gMieI{^v9X2%3u8Y3U~Jq!!u%H9*-axnJX*upufH8DxEa#~6Fp*}2PyW& z3o!{gOb!x!@21{%)Li!cz+oXe-Wr7(!}b}If|IGJa+x%asXRDH!ml+msz;P?&3KGR zMIs_1O@G_Xv#;>4nh^R4-ogu^_h%xQ6iUyMN*vIl-lOWO$baVk?1Rw%L4H&oF}33c zp?_LI3RhLmMEVfvg?-ZST3~v&VhEfmJcJG*;roXKf6MKMz`@+%i)dUVIb+6rCigA6 z{ar>PVxm-T4*yQF+&d7J01I)?C{=grF*nET(KGC2ADleniyRswk)bb)V?siJi5a+v ziOJwfKubUu7?K>2{FXH)V0F?N8F@(_8F>{$LUg>xs`s6-$2$rJWaMeD=V!kUFhOMv zx1YFmK0mh+K0ZIYajKn@BZne*z~NH^j}G@{oM2VgI4f9nRaZq>$kf4(4PfSA0%UWy zbA(l2;owBX-C@GF0fMPafEHHvqBO@%Z8TI?W}-AYyeb?jj?zF&D|t_6pr+@0EmKb$ zQ$aHtaWQlecOe*n9S{tla<{X!2MM{0()@)h1pEA_nVp8}FA}hgD2=X)8kMwzGmwgh zjfahcRo30gm5W9Uol3;n%v?xA=G{LaVBbV(EWuz$A$E2*H#asnZZ-#J3wBOHK|yv7 zE_N<1Ru}~<$ip5CaA&m#z4-(2H--!lWa@0?2)1&tr}~2lFmZ4Ji_*}*`l* z17!w-;s@~Z@d5;RS$PFbO;~x%`8ZhxdH76Ofjk_1+d>;X18RZ+f27ID z$Is2pEx^UW!^asQ3`M=e4yW?+H^{IOFQz+WT&YFD## z06W0E#@WhL!5$2Bc65e$(BD*l?)^c#L6{Yzj2dY2Q{mZAQMEsl zRslX%0dqkBD=#09pc&8%U}7%tw}t!@9pqpRb^|yAB`sj$g>eJ3+P}D=V*E>gZ~yyT z+$@2Al*7Tr&C0>a%E_U{At=PjA;iJM%)u$d!9m0R4~E(Qgp&W_ScLumB@>aq2>vbh zz{dUE28(I15Xt_}82JyG{So8;#n(S{@qh6OFzEkWu>rTwH+5vT6lePo%+Cx=0f{Q8!W@!P#;K-}mQwQmuVh=ws^@bLt0qU&U7y{d>Az)HaDS4gpL>hlk6XK% zySi?P9G8oWhrgnh3P#1Dk@^4chZANQk-S&XMG28}j?>{+r^9*O+!nG9!{z)k7X%M+ zG#;gp;^kdiD}e_3G-`7aJS89&+0k|1S&tvGnEcyqSqCkd3)Ex7ujQ5{D&kyju6k9F zebn}AAMfQpCj+eOm-9Yd>sE?_%$`-gkJ+*g9nYv=p?kK)&rW(E)1ZR=o6NBN+j>aU zek1CO)?>;Dc?ulS&ZeXdYdz@^w|Tu(lHgqb<8=Y7*mdIttjz6I4~h>xu}h`lJ2*=& z@@AOcA7;L<{XS%kVL-R4b@HLiVML`?)25?sJih<9wo8;h;C!+5cqbXN6zG-_GPrws zW&^sQf4-wu>}_e1&z$DDtZ4*Gk&~o71Ykc?1>X#Jj@FjCU6M8A@O`t);BR_VYC#BE zFMwD~76)^@jMw6deFZ=BZJN{!chp66aFZ)CmkO_=<4w@IX?#z9g0oyMzi{Oaj<5FE z2&sQBQpRB^0l2zcCHamuuR+1bILAm=Ctk3LK5g8(a#z>KE=In4-B-jMof?sf z158}(k%{e}%LJzjjM6BM&S9zu1X{Ex-nQ8)e9fTN!#y`c>EQKqn9;-fv-i z*Bmfab(HPE7_^YYj6J?5YK+w`HWtQ`?#jcAQORaP=alaH<$?|@#pS=ek`@gM&Epq= z&2Y&BmwCq=_qTqaeQkEy>z#s7`zOxETXisq^%ZFvi}!DcR^rUeR5dy^$IErrEDaoZ zYDD+|wT+acd*fu}lH9dhvCMt-N+x9j)_3$#0-1#+kwPDaRmI4C7{AEIos8mmBRL8? z=9DmvIB7D(hja6N{H|Y7ch}nH2(ZvAXvm3G>?zP?;?7JXv$C@AI%HP0_~mTwC5Yki+TX(x=CgvAlUQ=9$J56{lsy5C8S%prBplq=S! z&rp6}bq24k-DT?y(duw&(QY-j@2$vXd1xY2g!$y)iComy=(K6U?d#)>dt(yhm`fLj zGj7TSCt@;^oNY(S+QaPNu#mW9S0Y9!m|j!qvE#N1%oRVRq}q~btE*|_cX5opBwuh+ z=*HA=s={!Ob~l&2aFUgYJ2}@ouPFG_ z&eVtXG&urw?YP8+Fd&~_C8*vicb}vH;JDqQTgc8Ax{bP6BaV?&viqqdTb@Z4<1K+t>L z0SA3oYN~h#Ik`R$r)|U9=Grub7N3Jzzi^MLg)tX$ZS8TMmqV4awIIr&rtPmPg$OnW z1GMQhQ2W(;_sqBwbWDT3bCFwjBjYAEm5~uJ*wuM}F6rc;K;^q88dGhI8e-dJMVy9? zPG4S}Xz$hG!tRl!`t=2t?{pADp6Zc-XA3MUmcuoEv>=)9`;p*V`+~qqHwh;^6OD)4f6@d)8eG1q^2;n$}YN2vj(6tGC(3OsuRw z$_(6anUmQ2qI-5e{q!#=c*iGpOIuR1BES|WdWyH)d`{|{uYcjtx@*oJubeF~UG^Ac z>K9;oJB~usaD?N_cR-fONLIrV6>v&+Eq+S|ETw_MsYtHG4-xinVlcw38_*}FbYFutp{;58 zqm(W~S&i}I1$NA5X1wHKLTakkk`g>6I+A^B1$tb-dK7NLL|{j?W2-*n*PBp?QsufS z5%T_YhEy$m6pa?ROJWlI0iyf9f@5GjFDN=TCdzpZ_tZW5l;#S|~O8lGg8j(Q2(j5UTeEF4`K@(rkJhSRCK%K&cppEU88h`>Lkp}i$;cDuyd>#~C!zR`O{^E|J|ngg{0vTImTP~)O+^n$i;@uFlQ9?Lbba&ZnR{iN^?S>HQ#ZF8aCG5kz?92tu80#&^iH$sGhF2avldBDCqi+s+ zc_Pc{+)L1Z!(4PO(j~xO!1A02VmASHEK}bfx4()>5!lWG5g6U>x0&Bo0%2E=#lkYt zqYJj+bQ`rzCE1henF;wF#bHWcYk($C5nT?SYX4cjK=-Hd3z+=}Tftxy`OMgMU-f+`zx=?NZt75X(Pv`?m{6+{)u#80Kx zOp&h=1_=_iJAU!Do_#BaJMnwEiJNWSCEgm%7@Ae1EH(>loc|;OHMKOuG?>w?wB)qN z^x-Q}8{>H?3%h$>LkoYVVq7aGaqArxnSg}RGnJHjc#jFARJILM34GoyF7GZPgrQJZ zMa9NR)Zt4Z$;ai|+6QGnkO((v-yqU52Kd}&XNx0tiqPQI7un~UCy{sb824m-9p|Dv z2Jw$anZrMabUy_HYh$lIC{J#*-XE<$-5+f(Rzc0CQeV$iJ;T9gC2rAw(n~#oZ4FUP z3M5T6Ux(7a_K39$yS%a{jHba0&dnx8PBQey+10`gNC)Z!5??8E1>}^}Lk@b+gSOJa zivr7+53Xw|H&BT)S7U}#WgEw#F#EQ>zy&zbMogOA=TCW6`M$XgeB4SiJcwT<3am_0 z=}Srmops07Qk7{rRDv2Zsvd2XZ+^V|7}#jZVytwGZMC2^^<`Zn{y#C1~uw zD$*(RuAAYW8jl)uU5bz`0CLrQv2KbP(y*-DI?ALZ19z?ICXy^b+25Je$Dx8IB7zyqy!OC;XEu~dVK#~^X-w( zj`|07z2VAr0(F>&zCw?`711;FPT4l{rgK~dVjOhe2&mdF_9y<(6FjV*Jn=aZA$$e% z>2;xGb4;kqfSC;=!C8V59oHPoP$J7#q7qbYh6k53{x^!}&3xM<9AdZf>lqg-h(ZPz z*T+S5YE9L*I^Gy{EvFmWLxd?4!{BA-DKB4X7v=*Y_nTd%QN!nAz4C^GAwQ4pwnq=J zwv5d4rIZNBY33kN*v7h&i$c!-bAC-puU?H{Z^3g9 ziggunD#`w-C^jYK45)%U&EdBTVGS?u!9p80wVC|q5%Z$bA26TBlarizi5M7KK!6-a z)=5lB|6u^sIU!3XG6Cgpy_bfxuGr#62)O149^V(GDhj1!+s>sPy4T&D)L8kgk0=T2 z@i)&;K9jZw&e@H0)(6_mC&INaa9L=1RU{2=TkEJ}-d+H-SweDnK9nb(g(2dLeQ zq3<_d!O=yJbL&Q7i__WHqz#*mr9vcJ3S!UvZV$3h`6C7JIVQLYzfTCyfDvbWc}m`* zlHawO*Qf}22=Br8tSr_2wUly9w&YK%kQ0l^wvsv>D7vh=6hJ;DIDUYC!S$yd>Ezg- zC2{stsJ!h-C^Gl+$TUnJL=+SSZGQez)T%jZ?Zm=^l@JfinAp?dq4IGxy&)wptijnu zmxwfTM_eR71Ye~NeDU@Q^cUmzqxvlunn{@z_M_+Qtvt_>=TPg?k?$$or7vOW!t^h$ z@|JH+ZC(!0Edx=M=bf&Z=b@K`Wo^-Kw9xs~PvSpk zCjv_uO-eSj>EgC;Z*Bl(5GE#$8Xon%o?GtbfIaG0Y)%rmhHwKiWU}f-h#2Is?Py;| zkZ8K^oz=w!*#@bbZ&1!EfWm!8GIRclMEIK0*td_xZ~^4sP_-a9#vi`l5FM)mOF5(@ zTGeQE7_l}YH~rl7kI~4KAXYAAUrWMkbZiA|oY0%><|fuTOr3aC-o?#WU&-t?QWF57e{Sg?&o#_AHFkF;yfB)_H}ATKTU_u z+kg#TJ5#2L(~Tc^!HO8D1YBBa=HD-oRpLi@QVUdGCl#Sfy^oOkz`IfO>0zaIZ{Dm_ zqvNu~6Rcrlf1N3chwdQ)?W(oY(grv??NY z$ts~Glrmlz<=m^9VyoLtX4BMOn3cD%;pR8;cv~F7>4_5iRe{b)^fr~jIbIp>M_z*S z^C`U_=vo@uR_SL6PS?n3FGw;(^V4D@c!*II;k<1SZUde3Q!KiB$(*?nsCyjh1S@l0 zZBB260!vClOE`R|^09D1Q{8E(Sn!m^_w-VwENS{BLzE&=j0hGqiu!|&_?uUIt-_F@ z)quQeK7NWSfwdpT+Dv6Cmff}XQVEfa@QvG0>MZt^daMDnuaWCFkbSyvi)v!C7>fk# zWyeEtL)X!}OGD-DRh2YLd&`xTRm0%p^W*EMlbkj`_sK&C@bw!R8NV4R+R7O>m+ECh zZHphcKj%B>>++S+>a}ckCb>^1XcjC>m@Ec@NSe4}swJ0Wn#t+PM%M2yUXu0jfJK(& zUixmU25x)&`gTT^^oy8uXM23c_1qcK-Ap1+yOy8`q<0o%!q@A@JUkwHoyX1MET^Y;Ac9f$f=N>ByYNk&wst~6&KFLS zJ)3bnI$HOgPMslBj=V>0s^6E^Qudy8=_IbuTTOdE-VfLm$4Ex7$fK=p>0oDW>7H;X zd^I+qCdU`w%M(Na2^pRM z^Incp^pM{*KW!via*J+SCDa3Zsg^C8)i$b(o)(02qv&Yvn!c=9txT$<%=X1?4O&Uj zpqn%*E{>@?z1=wH`85@am^5iNMN*G0*{R~WAeLGZdcNEU89GKt(wodDJwkbE+`g?u z)L~uFR-kx*n9<;DA@A*57yTIE;^OP=SQ0zysnlQpZo|VrJ-##%y)KY21OyvMYff23 zNTbYvh)J_nB$mg;9WmD3-g{BvX5Eab_*Feca}$8w-13UhTiM5`$O=48dy>egiK5w3 zjaYKe=7m+y&i%utS!|okWZlA2(pbiLOTWYxkj#@*R{sJPLFpA3Oi)LtQo@ILdmv%2 zvP;lQazj6~Fs+v%9hOqDdhiY%dQhaY6)rriymAy}XPMO2+-cob5k`IZyphSkeFxe9 zs@!c=RGO1tkr~>^lA;v|W>V;>eVa6-n;4QqT3I^E1A7unRrzQ6@eU;N5LN~VjjJOymIIFS= z>1gBkIY~v8gwbAi5rbyFWjI8Z&^Y1*C>ho4O>MrMn)lS+k-{f|>?5xRiEu>rXv_+t zhyjPt<;;ARlM-mzP%{7O#E`{3iVV zk>fO3p0=kWQ#yV;@tVR_F|p7qn*Zv`4L z5dK+arf(!LEfsv0Tu4ro@x8T`Vsb19cTS7EDfJbG*V1FL>hN{G2#8m38|i!6d^=O0 z2C$h`&$>h$WPlCLV(OWzjma$nm_VJ#%PK<(tP&LSv$?4B>|aVV5X22F?|SGa$`USFyF4h~+PK?9fE% zUmM{!@Pa`Ijkn)w?}m+x{B9d87M&k~yH8I~iLr^HX>rj6$m1j20ZlV(wl;?Iyn5(N z^l57Gm+qTraMlwFS7anq_9OsrJy|v*GZr0VcW^YUNGPiNFp#SujghH>2XnP;*-sC!i23u-8$qFqQO>OVxF06SIJK>udr!ijG zCMKe9Jz!=fL>$50cj&`3SzD?Z4MFcwffa5PL+7h*j%Z`F4J{2ZeopFE-|aS$KvUVv z^M04-{MXGdZeFf*6;igjckD8h=XiB2a*+Bwob^PjZj5{xk0_5*+0d;r!>UQ4MGg6t z?vn0w3AFRreKx`UHPZ6d`lEx%nemBV{sk&6+W8tOZO_v_)iZ?x zz61JSonG@As-6!$-qf#qMN{3bn0;zk6RpTG|KU&-rFei%0g{7^z&=qqj~Wc_`{A5C zK9kX@`6%odB@H)hUA(;`A3GrRmbU4zeiowC<0=5FfFgl}rI-y50p*Ey6ze=Cdw3Y%3{o=3A*zC@SeyrX2+xPl^%{;+uhkh7Z-$Q_*)@vsY^KSe zYqC+gvr|3kf}rpe`Z?P~Bn*O|e$OvFJw4U%kQQ8}#P!h{FM?lnuXA2e3U!xVxo$zz z!ib}CtVvQKLw6o(PWAhcrstDn?W zK}wiNvpg~(x1i*O@!0)_=0;O??1vo%bN>8~wQ5Yn7_t^qp=kow)$7I3enjnWD-d>f z_jNwshn4HMmn~#sZgV9ZB76gAQNpvJ9mb3wF9D$~x)8MrXIc~b$djO3kU3`AO=}$G zul`cbMzU*6MVnYyK>)ugu%wu};{|3{w#kBI@yn-c$|q_89N%FOmftp1iloxyqK)qoZ?}`i)55itFT)uiBv-rn|Q5`j|9CA^l(cV zzt;bFRCS16s#AWamRoyj>14}@8yFq_6pPcm-#Hogd@<2uz(1PCtyfw)(z^Rw$f9ms z*J#L3tO9MQRQCX zrZ30$o6#8_qMHD$C<$?~4HLhA*iN2WdT(Grz6(azp6WN_f}@RB7|j2QS1|U-)F!Rk z?Lvn4JG;0TE$|@#Vg)P4!cwJ1h^aP{^ZTXCS;x^ik(05m>RO&E?;b_i;cmEs%SRk{ za?tj$f_r8iYc2qm#^D+XTo1>yJ2==qMYvU#p(9A=ZD7z*_<3?WO_|ENZFA-3}kC8yR7vx{~c|>pib-fhipa5QQA|fLLSeivkw>&o2ni48))qD+4akve>ZPmHGay#1O7k7UneltEP z^3+~e9DRiCCnl0;`uqj3Rm(r27N6>*TU62+d4yHl>ic;#F_hQ@yCwxs40ZxSWFMOY z2rp){uS`_zq5pFhPiIn~kfS9v`LeOGac5_z9b{mC(|+G|*K~ZpcnN(TUX%R|5M4Y6 zsw{yFi21#yrazI-Fsa(vQuFLezKMqBNQWuVYi^w1{>)*Zh|^@mI}wdbvu_T-ABugG zG2mEKmoz$x<>}?MYgmn1s-H-FlYMZtxV7CE!2XUW`m59N$`^C8UngP}_J`&_cHL>8G~vKb>!l0y~VUwuBf1%s=3@xvwBaB;j* zpNK~A>yN(13wlg>zeC3Qxqq2-+YuygLZd^KHlksq6PC-!!^h|UN}!fC|9dceIMo70 zzvf%?P^(aBP5^(V^1;xz!AQjl-S0K|DH%uX(hL=VnPjWTw5W|{>Cd!80J!V*SkByuZ zI?m4_lW)Q;z|*;|>R>MEC3?xb%kT35c0%s^#??2~1v?F~)l2NY3+siGo$g0qjHJSW z9e%9+GRNi4Jiqj5Ne-3lag}smJXzwlaJB3IGSJFpm3xOS|q;o z`TV$xASdUsJM|dujzM{Q;!x>6p39N5E#R8$p@WCCd@fBltxFTJF9fS@nw$oU5W%~( z=riT+Bl|o)YL#(8?Up**mV5|9?SFdDaY0(Pd|>gND}@DC(Kn%nnz8lC*nf>t74K5v zUy^5v@379&UpX2;n21EC)VjIex{CZQX!=VD`0^!Rq)Lwoe01@ZxDH= zf#WVQFv8(q3Wd1c${Fs%X8TtyBGsorp-+3oL$LJ6bakKU8?&vii+Cx4o8weZA+BG% zV+S^PPLAO$I!0B4xV=9xaOukjbQ z0$*-}dUp#~Jt+pva2IL^w&|18hjHQa69aZ@+r)y_5Kb)Pb1zWHotjfg*)^cZWhmA~d+JDmD`9 zBU<)xaa5tSaW_*us&O7O{+5Ql*)u$f?HP_CtC|fwlwnLWQbCS6DKP~e)+Ot!m0lCB zRq0{UWWvXhnjW-k>byEE4<7s(MZosGxEP172zBK?K~eS~@&a`tg(URdyB)#BAq9rA zj`JLSCTD1y-@;=H)CWM9zgUWcpJ%qXK3-N6krL{qCcnWr9XqyvJKld6eQzwNJGmQ4W*4p_O{U+ zATl72p?ytnLr`m?tq)eAOJZO35bH9H=lPYtl&~TF=C#N}H-^Z~0oFW;_03*V+wf^0 zrK$>_@W_N}#D@n;6${wS9GTK#9=<~aPY?TUmEOK@EHazZxu;o8W-zcTL47u`b%j5; zG`V!UW&0M;`A%DVE;!FRXNWS5?;gU4e!evVbaxMK0B!ky4$CO9)?&d>nF$0Wa+Q?i z?h27tS>%oPPNI%{XHICxUK zu0@z?yUMmOi-S44ItIS4^GxT%6{!vxtX$5~hTX*v+Ygu!k?^Lt3xpIf25QlSLQtLw z;U|Ad)6FnN0>gw`TT;TZKELE?@c#JT--B(&u+|rOKon|mt|WY0cr2zM^u_paB7`XP z{dH%ESx}=JQ9Mc@e(N?$MBO_QQqqs02Wxxx=ga2mLtv}Rb=&EpQq$~DU*1NsQSqnW zM$YRf;-UMVw%%t7usx(jrc)!C_o)caZ&bJDw_)-Qly)^I&&ppHjTK2b z{lp%%#Ty^J|D4~K#1iHWDEQFR-Y?Mh*b$}BTex-^d3e$imbsc^;WokBgk&%|#`{fK zNJWJ&YP&%ziPAkFApslER7#N8FlTbSw01+(qg!}8|N<)`1v6}~RQ=X&wpn@U-E z*T$4}`aW;{p4pc-$5y)*Z=UwmT}1J|mW6VsvNau-e01;P+0;r^b4o`qDG0Os8CjBR zB_!55yoM4Mw%=&RO}ymQvng_O@&@+w)cE2e*g~cQ#tb0})0J=?Lbjh}`Ym}3_qxT* zo3=J=@2o*fEN{x8pz#WkNJ!omA9*Pg0XXn?T~=6Bvu9aba~yD>JdZh&2-fpK+ZXrh11id3w0twBRr@B6qVuc$B@TaVDN=VQUiOv&>-*{WjN4G*Xk{*!^l9pq+d?yGoHJ7n8wx2_1cu>DZ!{ zCgMMfee$2OB?C1z+g>_KgRTyoO0+3*?)%u}G?MAW2j&aB|f>w{i0>fdo+8b^=IY`nlu)L)*=#x~q`OiA(mc<-TI zX>3aJZPJ_JOmhPgjr@u0z=gHX#3BMibd1CzdNI7hWBxRGsV{;We^Cql90W-YV?iv% zpMNlFg%?T+OVeUCX^44%Pt|jDsbg<*-o2yME5w|foMSGcNM6lIuvi05{Og>S4Aqnp zB%6-|gEDkqdd%S+JJ}Ds0|eTW)}9v^7Pwr8Ei*1QJ1iB328^MK(iBC|R@TiQ-z%?>g#i|eQzrYIHE`R+^^QNGb-rZrRDpdQ z^^X2_=gF5%ckR@Lk&7PCQ=SbB*aph6@;;&MFg>J+HBnj7b^;8tkUu`w$Fn(D1YrzD z`{}WN9Mfx8>yHU}zdt7i$=@OuLtRWX1Y6jo z!AHxlH3U{ZB$WWylD3QmP@+U|iX$M=F$EoUMUh|>6;2}ngNU2`qHeN)G9rG|leq4m zDXz^-XTO)06rx3gAetdhJ)>@%%x~JSh)$nR&hImEEY4F7{@0PplG@r9Gr^{G>5P7_ zGWdCPT@@`slpFLyj1j*Ln4`@roudqT{`7B5Q|do7`CKq6 z4k9e#cJkIL03BRUzLnLR4o0~wu6nNhOig4m|B#4A+vP{yHK)Wv@;#G>1vNtP(Tb!- zJ_IHehlh_}9Qn*G9sR@hx?B^RR9^<#;|2x>_Gj-`EPU5o>dMYd(Y+T<=eNo8B<4e{ zvb#Yq?>O6r(siXZ>W&FwIF|L>mmT$}{Lb>>3jk8Wpi14zvJ~!U+w2eII zTXc0h7;qGb)`CNbU*R*?7o{mk(1f5?8^HdXGzKk~#VqUDjYV$Ha=O?vHf>G$Qi`6# z>*ZZS;KK<+f3!Gx5=L{aT}R?6uTMUqjyGg&f+^nU0ii*LM5njQ=K-!94cXG;S9nyk zYeLd@#2m0h!M?qe;DYsNm<<%i;>*Te!CMk$G$G zX1Z*7+e7oNn=%%+`Ri-!NDFr+afE~3Pf7H!jStn28^R>F$(NSGfYd|ZJE9}gMirXF zOp94CoMaTXsgzbyLRm6&M<@<@y=F8g+w-Fm9$C=8-6T#lZrkW{eUJ{W8D@%yEv32-V}Rb-nMfJQo~gHg24p@}7Gd(AD8=1y1#`y^G38Hj!*!Q; z*-n03a_0}ANkii7!*c<&+mERY=sv^XAR!(@KPVL2ypGg5j{ja3D5y}~cMy1*Exv=MfMp2`htNon%l&#MD>VnMcc4#>F7S}6vVfI@ou;Q*4= zR^L6Dq??z-JO|%vYU$CD<<@u2#XKLV*N5}bdqJ>*Yueg#MbYVfxt$B9_BP|D*wR_t zY;5AQEpq!M;GZ_sS=h9A$C7t(ekl%B)cj=?)cWmQ(pMxECu=>cf%*KwSvYpg?-k3R z7LuCcXlHY7I4ZU@19t}0t!#k) z`7)coguyg>&-E8it{=a#7tq)j)ktgAD!?uQOKu@N>KZ*&axAlJm`JlcV5#2Wbw-8MRZ8z{`2|CUv6?`Hq%_UZ42mf3~PG3u8cY*c-S?l}@XGI$>#egqRy3P6Qs}X z1++=~>sO`5&o-;znfc}EsbcBrt+qcUKW;5!kCjq@$tH1CjoQu!JutfgmAO8HPlH^g z)8S$J^o~flAz`n81h8cn)-7_oLZGcP9NW3@by#nWlU&;yYC5$lpB$6;E9KL#ci1%7K%?++`SQ=eYjsISg&x&3#Eo#OI2CtAJ8oTKCbL0&|qMP6U5$ znl82nBS8Dpy8Z>DbqG)3KAun-Yc1`|9 zF0+-+#8uVc`kBWbZi>&fJj0CX3CdvFT_mh~bs*~|CI>&EL0wuVnlx6R+u-%vRl_%> zfxiJ+b)T!dZ|7|V@O|^x2a`^M2iM|Vo;1O`2YCV8iKOm}wzSh5YM+3|Dh+zY_#}>B zEZYolu@nQpROJw)CPOK0?Cntq4J&h0s~BS6UGf^gl6% zH#*}Xlq;F@ANCx(u~gU(#fWn~6xkd`A^pG_EXK?P^aHK)W48Cdr71LaxBiK9atc?Uwm~=>YkG|IRwjyozG8b7$-W7e z4Bys-v)yZJv{SdLESIRtLQzP!9bf}^tgv{2;(E=3cOk-?HMr5|1&l0r=7VY0$26XejT@5qa)b4uaz# zWcmPO&Ra3fv>(TxQzbhAAD9`pWx<~iNrJ$@u{;t`bi)g-aFDQeZtuawW;M_JjPZtR zjB@a`e{<`^!I0*9q0X?rshL4f?o1dVvx-KTma0_VKSCog;5LaVq6^h$9F>~TQPh*s zOyTkzO-Erb{hD~HLXSlUkE65D9O6SkwWLvN5|-_NGUdD?Kc0I+cyxBORe35r-cw8; zm?HOG=u$78(TGA{y9)o-#rt*M{y{J+Mm=mb0=@7)KM38Xl6xXftVbFWyJ{Re!?@K2 zd<6@LXV}5LB#w-^s{@9?@8ocaiP3Fo4BWx*0=Vw8W6$GGRiAR?Js$T+mLI#J6KCfW z?)%i%M8%IEfb06xU!eW+WoB$+%I_*MUnBH&DlK?#Vd^(F?QR zI>Ds(*i|rS>3aSCruP#orF&oe8l&7Tx`PD~SYH)K%PsuZ@)8)~z|C>as{SA0ZQ#=% zr4mroq1{dIQb=d_8=@+zZpFRZWp)BBCHMv+rFTh!3|Ek@GIr;+Je0N=MuD0cxO0+$ z>p7t;*$jS2?qUenfcl zALx(59pU!OLODQUCXuDxI9R?rH99SDE5N9yYJ>^9@z3;G*DM}qX4-rgD28+vH1ACR zP>p$y3>_HrLg#TmZKg)kZ1TU3oneLzRzCwSdjbWY?>NSf&r<(#Y6o@eYV~ACrZP2u zAV03(u2p0TO$R;-i6F~V*3pj7!b9`PPq)?4iNOi~k;!#m#BcgxJ^8~1U-!KHvAmuN|o?Xc9H5PZJF;?U+VH1Jg>y-iZpgEf;*!&Fu##h;G2LJ_+ZQ@XmQs=Ww*!1rLoVD-H)S17nmoK3petdh@B*4GULu3v!W}ctZ#ktAVu) zHRhsbg^Ck7l$okkRs=JzFR#8&G9n|K$jN`fi3~pQIhQP4LjA(FuWVGn}P7(et6}fnvd6V)aq+5h5EH*`L^-p;Aj`*9d&%T zD~!j-@B7-FI`uqvE9JJ^v5~gl2!3_Fb?io6^8A*u7oudXknW>+xW68_+%vkym&bTv zBnvbEoA?bz!NeEX2wbZ6jmlVa?5`)0pj2!6n6Mk0W6&bn?wLGm(O9$aGhd#2``fI- z(S;FS`YNXqQUrbG*A7v#u!-`8rk?ogi5$y zu%~JXE(uvJLyi^L!wlzl3nn<+s=4#JOsi90K{5H4L_vdEN|@TwX!b8{uCw)efMUabNDDC4{hP*_O+WfwLh z$LWO(sT1dqG~Os8W3@E+cuOc|t`!Zngl_x}FT$Kz9^i>)t!H}fL>2u``dYHm7^#g> z(b2#|k&p1(NulV(m8N=zwCP5AF(v(lW|b{fd10l6E^JNiZpHi6I1|_=sye{2Pi?N9 zdPIn*F2FAYfsI;6)$_Xauqa~ZS7qfd*&$mehs@h8%&9lx@0>jZ;HMK-H7T;aIK1vY zFVOSO%hJ6uSbA6u=mw@6^bG0LihjW;nm3J>6HQs=Qeeej=tZb08LWhu56p(+$R(K!4{x`@ zH0!44;XDK1SCN^`3}{2sk25_jQl2DV;t2z#);C8DQE+5ZUI%7UfPY1xRcp}%{OH8|aTncph+}QvVf*1Xcn_j#k?L?rXmO9G)3XEqK42#gagd1>qI+7DxM9)F!9#){ed(Mi+>L@yTTbaPA&UJr6uF>+bkl1mo>mx;njEYKTF*o5j zv{|1{Bc2*B0N251R4^-|l{1fdk8(Va*m$}l^Dq;sK#_z_LrbfQ|*caWG(Slc75?!A!_PYLW zc3Hn4-|=t87tVl{mAu#jc#fLl3Bbw|`?Mi7Rq1|Lhxy;`iK3p;nl1GaG;M7@rVO>D zI^IjJR{<>Jn1A@YNl<&bddz8axE;m&l|$;%k)F!|B)|(1$kN=V*ZSwRN~siFlsmN9 z#f=vR|8EOaE6U>h_`B1I)+_EB-@%I7nY-Zge3RhQpW?R}YvGB?GzM04wlB{C?G)c| zl+2P;p;%(R9jB}v;{=96V+R11xRZ{v-CHd{|@?$@(Y+I-RelJ#)4_+P+XHsGP&I7 zUT#&AFhq$?O56Yd?0R;xwA)x`7ROkStHW{(L`U zrNNyR7!WGnAbD8r)q(3(NcZL3v8K7Lg*-ypDS-2gyv}RfcN+(rM{x^LYWUH>!|+0s zR;QXS!lPk7a6u(y-E;l-YmL0!isY`J6CEw}WfklMD{8D6%ve3e_&~Ir5VKQF-Tfya$L=`(s+ zesiKGL&n_nv@ChSD1BPF2IJWN?5FdkLil+{l|)&8ve;{tTH1S*K$ribu6^Bo&1JZ)j)`uRUiUA7!hfiW z9GjUbhWw&prWi((R4q@e@t0tf{h?rOcGz zj`@+1k*hz&Ndv6NLvmrITC4FX9xTaM_{6oJK5t$Ma?knRPeaLH!vy%>DF(LE{ zN15LWH!@1(4#hUmJkFyGisIccHPgj5cz;3)-ky04$;E%~OptZx9CjTibOc29U0z}2 zT^6g=6z-)j7%4p_eXg*#crNBWgl}r9TW3$!cEEFT;|5!n`bL)y3x*2MUM+ann z6l!wJOpkOcr9b$$BZ2b7)$d{GPX_Y!K1 zLrT-pMjPj=1&kl}5EnU>u;b;jxBl64|JI;4HT}h~KQtiLqE`hS@86_q=h6f}D2f$D ze?|SOL_p1cO~og{q`B|6#6|^bn|g8(oPZ_iZ|W+URr~gxeDbcEOhl@pusR2VZHg94 zL4oAqY^4~>k-#A2vinew@9hq<2>P7|ZK{>ubScaCU~ilc<<4zCqlz*w_p-bIxA<6a zYxsGkQBm`|NW;FILDRNfE-za+Z!A_#^Ik_ei$2YmeJ!E&N-O=GV_AvSy4`i@>HU~O zSOp-5aWgKqPHHkRduXRXXA%QMUozd0vCF5cB&U^UF&&CdBoKrgok*nW|3E$3|EI4p zX`n{n@Fq;xb8a;Y^=?z}%;@T^mW4BCf_XSCgD~(tu(#Qi=Tu5uE{bEQ{3yH3_KO69 zpnL+iI&Y^S(34@)3kj%*T{fvTGq;qK-y^Kk8s2wF_62AnNfb#3+~-)7>U^9U-jUCy zhsz;KkelEY2QnLEQ$ai#2rGS61y~za*7>$f!8i+Ry6M~i_bctOq<$BP|Lo;hUK04S zW!Ukw82SNnaqw#B)_D{EO8ZIn6;y|mAWfWRcTe3*Dv9{UyGv*q1-aU~&5P;)I?0x( zs%^LJ^E6~+_QfMQinhgzDMCcp_ER!6OQsiDUz|*mnTJQJ zaq56JF@!udJcHlqCx+8|DXpYaH=0v-$>6g%s>dcMakAwRXH>tdTF5)}`?ulcv4)J2 z?N)wrMo@&QN@iH%$ubTjD9Uk#75N|6Cv5~pBIdlEro(Cvbef)$#~NjGHi=Gpp+Rj3 zMET-L7ov<$F{6@M>X@Ykqvk$uaKaj_=DL#Pw0QhL>HLB`+v5qR&10xe<9&m;GwZ%6)4 zzGobyU4au%jW?&YB&6-1vxOtp+l%+Hr;&I^WXqZkfAU!3;}iS7<*jJnKno{-89+$= zs+0bWy^FkSnT|(-P6r1XxoypjgW#S3udsZ9uw0|cMEoL4&1gX2J7R>2q$+|0V-l5X zM6&+2wJpK~&kCf560gysH6y4dQ*sN0K{m~#Xp0Bu5KSn9^NK>j=3P@mPmzFN18QVt zXGacK(@EF0k%n?j{=I;hAEDnC7u=@L(-Rb!N_AkrR5y&0q15v@*!FI;3#eNB4uv;U z7)TDsU%zBu@G08Hh{-^!m9IKrtqV7$AGYLh1IW>%GPMg#K`4%vw3t)-J6vkNv;#J4 zhM20f@3Ye{@GFA8>bV2svAF0OIi;}R3?`_NO*02_#Vl-N7+0~U#q;saCN%Q57ZO4W zO#~tLKLIdwUr^aJs)0BPZj+6>Niz;DD)ebjc)!$I(3HQ?LQeRe7Ysah;JkW^d=6Bl zKR%Kth*Qn8GZz^A@EfLvNI#zL5rqz{*zct<`qO9$#(5rbxkiG*Aw0&tU$C>zk*rON z*qLyn>G;;NO0}x`8;A{nPQ+2#)uQQEYoA0Rr|H{FOk{EqlezT~KXzpk=976t4e1fX zkTg*^7sONSYQ>WbCpH$Ih16fqfVH`W7S(P^+z9qIO`Yr}$>>8>KR7{Gl_2K9KYwD{ zk6k*LFsy?in#w+zEY0!d&CN|sA@O(F$8|?WjpkXw12_=K zZG08#^S+jgnF@5BBov`X^~*S>J0cCjl4ME>|00_bt**8jDEY-QCObyIpvS5gl- z`j=bn1%KdWMPn37+*T|u(fTg*BdFRauz{Dk-oQdF^%27AyZz9;;shgSdYZzr=M_S& z4p-u!jPllKC)y_Lra6Bz%aG_}_uW&#VWipNd%zb}aI44lkL%jHJUyF!CcXjHv_HFz z?aRJufK)u}GO{d^R6;1rIr?)^VN<(yeU%hx*8v5zE#% zo*r9|%-5+zKX}lrg@8?fxdXZ2#HU`%ELf$`|Ak`eLp)MCVFjVTVuf#fa-yG8F|&wI z?wgp$1L9v^h&i@zIA4FOwl+Lo%|P^hPMZd9TX~GP6yferX&N5gHslydDo?TB-Q8Dy ziuhH+t~PG9{Ig5IjC^htq$Gr+t+LRK67qjAv$K<@s~})i!UR}Wo^#5h;LV5Zc95m| z*SS~Gknd$$)=NIH5b&}tRM4(xWvh8Y{~Sd^(AEi(FadIRUD>8g4S!xl*{v1!h0+cH z{@w}>$w9U4^))iCxeTJ<5#tY=eD)EA%q^j6O9bTCZMa}(-?~3N>`rYUZRX6G213H~ zBk(cGI7Br=^4GCMn^<^Ax`6NjhtT8W`C$r_CAI*IA`1xxud;7kKZm;XH?9T35C*?g z0W>^M_IF}UGv$S2>k_|uegGoXaf@Hk7!r5PS=szE4!x*xbdOFzIjIJQ zX**bNIwmIjPrS7qZbA;DBd;5;Brv8i2|V2&GIe_0CwGB&T$lfRD6Mca>a84LQM*_p z9p{>i=g9-T3}iR#)q~KBRXt5t9`KD0525!POa@bEs5gFWrt$5c&fuAgZax20=`1Tbyy~B=hzf0ok zDLFdEx4QJ|Qv*0fI_JdISy)!&wuSO~jvCE|S%W?o+ltkg%`GfoFflQoc7yMuGqd@L z&Ni+cc&z#4>H^levr{HE1VX94*$aYyI-`>!`2{5!l- zR_V`lN@ocmMdl`IXcHA^Fx2S$+(NtBOVMW);F)?pa=W8$8;GbcHgMO}HMTIPCL!lv zy9vlc#6YBIWSY9rNEp}yW_!Z2sFngbZT6<5+~|c)4;HPnelNx5I906=oW!w=9Q|S4 z@I77qMT00EtxXnN{BRuH#$({V5(QdPG&A!-*w~@yxo#@^pbgO;tYKL)4K#)K6Tq?y(0|d4su%oSLAEM;YWd77 z5e{9hS;dMBy)jD>))&2bO!T=1-j?&(ocKCTSljEje>6>YW=AZtcEc#2?t5rqe9v|S zn~5Ym{m9Y+l{RC{W=%U6W}A&r3^OX!lO_G_8w@7xcqX85axntJ>95%6B2npZ7z#OJU@GVyQH#yWVESpBV zH=0DO02U!t83^Frz?bC78#sz8Y}+$3XR^v)Hdi^)Sa3Rbf-n9&z4xu&>u-R|fHP&` zfxoY&0qfa6Tyrz?21P#Kb6oHD=q`M1+zCoodP6R4tw)eLPZQM@YJPu{(Kx(MOtD-B zx|8Zn;2ip{as#X=OLSvbvUY7ING0g!^<$AJV|ArV8tvf1)xgQHu&xd_hg%4B>J}o5 zdFr;vBtnl5o0^|x-JO(kLy@B+$Fhv!Ft(3dNuhOP_*K4B{!tQ-=W17CN&lDg5wg9t zM0vlOJVqo&t{P%sclX}f0{h75Z-T80XKWQfl6t{$$RotVWlbf3nQYcn*x1!GN)uO<`#-}QgNFqpHk+w z2X>|bXL4vE*s&%9D1&UMq?ggEz{_%7)He!}7_W`Xu|5F|#NJ*gTIvnkTC2Ee4+Go( z@tdoNF=*>p1{W^V!Tf&ta>HAc2m$+;PAOv)A%;l5&prb0XY_h35;V2M)AQ1ykTfyA zGuBe&dN+;Lb5~P90@EM7xcxTE%3Q*;s+iC-vvG4usEOMhD0H~bH9pB=dh#5ON1|J3 zG`KTBp>>x!y-F939=0Bxv6X7jGcNw4p8Km<+WrZnDIU3!dM>Xk6cQ2=LICMJD=W}W z4TiRZb0hZlN-;(nFD#S>+Z;h`G8OZmbMhC472o8?{Dv-p0&Qpj|7Dr@xZ2HHXaHYS z19IVIc&LI3NWSlJHKsVmf20i2>fm5zwY+#I-E~-sGE-+G_*&FCDG%`NgqqNM^V9Kd zlbGd=<0=23?KHmjssf&y^sWEmUH!;oGEhr78AibCOsYn0U`wTE*=+Us!4iG=PdPQ} zfiGMXT|*XK4I;S^VEw8Vq~~@8csOpAV6vd+IKPnW`qF%#!vY|9$Z0b1q3%%idoCG;XJz46>$laC zO=fmxWXH16T4W73gk}rckGd{%38zqdBwt0Qn`N=wuUeM&S3wGQa74yXgV;;tC^TG# zrS(U^OW#JMSo`o8R}b9%Tql!?5nPFi*PXTnHb1v9jbB#%c|Oj)gJgUbLAp*< z8g1nP&f7x=KPBuVbs1cUDgy+70k&4+5^^D4N`qjhUiuG882}qEZ%BW?*xmg-wubp0 z^14Ppsm!mF4)=t1jpxC?gmFX-mbPE!8ow+mNd}^YT&c%tXdBgzSTU+gqq)6}4EZPt zlbYizg0zU=s1h+`FpB5~c5@1@df(DI+bEniBdG$8jt4o6J1%LHxD-@x0gF3O&$q86 zJ(!{1ZK&qpL&y+8$vtrla<&Np-+?@+_bd#L%vIeED?+hm;4)d@1euBujWXTx^*UwV?r3C9O zeB-WOvLmuYaC(apxO^Gg=`W@gPNG_)cAj;`bMU(Z?(UO{G&xh4o^ewZ)(z_XN8MDc zM802kJbPcPGvBMrqbL&S4E5~n1wMfU;tgv9G;(qORoBTYoif(gC3VHbHS;!?aePchuq}&dxW+# zKD;bsIRoBpH%3EN2}AxT6&vHA7~WG+FuJ*|Euw(;dH$ky8S0&}NWc{QRoQC(;8JVg z_DApY)w#3JeF7o#aA5@{@6{VQ^=}hhIK^-f?P}|=?rNlf*aRFzn4K{Sgi5UjE5DbQ z=`3JLl4w-N=EdZ97bcU>)?M>3MyVh)8TO%f%p@PU>7sDJo=MK^Y^4c4o~_!j69G%{HQ%6hXI7J zQ1B3&Y=kffTII>DEbIKqkM)9&jAtq(?~^#Sf*Iq<$u}a#*He8jl7IC<6Ms&yCEm-I z=xyd1zzK+G6^1&ba30r)Ak7XorAoWsHMc4KVJ^B10|rvQHS!}n6bSTX4gYj-y=0*t@RYkYWZd@@I>d zLUGkfhtsiJyN_%Ek&Ua4*?>l7=Jb6jnKbpLrkG-c@NnoOkAV2ZL@=L=EpHD9o8)BV z@}i6&OBAQ3s=%=j{W+9dLF1T&NRGho*t24VCz~R<%!3qXWlE4$YP&?@MLFq;i2*GN4H~zXX%xfrt!Q+X z`OU5K&x?77?<%@o=-RY!7NqFI628{P<;QF_Ga;#Bk$*Q6 zY_-LV(ncQiG(^681S7wVe3Av+EnNXf5T16P<+Cu5aXW=~5_qdvJB%Z=!fs((lx|8k z`LH&eHnmwyH-h{5n$gyEKS5acxkUOtWR2m&@nYuh`+MRjNVEN(9KwF6cR_PhN4+(n z1u4&GpM8n`agpebXRN%PdyE=sp)tJj^!MJLhlWTDTc8`H@)=)iRa=|${j_MoAf8B@ zf?c_)4n5@WF+WfY78d`9X6m#76m$S7Y9K|#eY9^69Y>}p&Pz;?jXQtun)Yc z+O%!$A@m-C8Qz67fMpRkb@Hfg-XZvFpBzLIlqd*Z58~r2+?gY&)zQwU0vlPd=W`Fj zyd7cf&6%jL6k78&d0%LszLaVu$Yxs=JpjLNBNVcO!f+u|12b%h+oT= z^P1fU5Soi)%RCKz>yJwf7IJ7a(QT#Q|5nNyU_lW4Sp%ORd3g@=d(?)%_)|+8IvfJ( zDIdP=JJIUfN18qDrUTB~ID5lTvL$S!mP=a(mE0C@;!aALZRO;N6tS1@+|SOvR5dEO zSPjmh9p3&%Y2t(tc7qNtCMV#&V;E)MlGLl8zZilTP)`Eysq1e#O>B=fk^MNv*^KtYcCZo3^&L)OfWFxAs6JaP4P@jev^WY2*b7O zoopN&;dBo?2dUqr`x!rW}gVdRJ?k)iPbu+FxSugVjqPEL@) z?+d-J87A9Wffz@ky`qhN;ZSEHhJyTe*U~lrN)ZO9nU={iWwto`M!d&A%~CS^t~Xaw(Bb3p zO!%U#Z^-0zXe$<(hJJxNNoXXB7lkmWQa`TNpqUcyvMQ9Ej z_2NxTkS`5ms56+t)frz)M&xvU_*1!Eh9-0dANu*{;ap&iwoSN&CZ6(UJ~7b!_Z>+f z`H|?SBdVL7{z1}GHIeROxu&UZS66yzGxFD-pUEzi(1Yl&2Z91Lwy}wq;RJ$693vbZ zR131}ZCahfY5Eb3NaU$=9#R$gB|q%OuTgt`XjQ?24%F0J{D|`p!Ai9SezJb}k(7Ajb)e!ia@@FAJxVYgd7h zcxtU$Tz;=&p=~%e@$kVz3wrc_nwcW~WJCadHhTvU^{GA?m597)!*MRX8;~JFg7_e} z3XmV>(;qqQiZGc2|JZKyTV>toQ^NN6<4M;uA~Ax{BXXO+jV@Qi9gS-L0mo&X-^w13m? zm=jc^$E3|**+NaD>w+#F@C5<_2JOEc0c+<}Qd{sY@~8u>l3!`TvqM#h%)J`$X?4@p z#L>9kuh6#rlWKH@XNP6{%!Ec`7JVrwK=^B0llbyaLnAYi2Nx)@nd$%Auh1a5RSmn!F(D zQm976h4_d-@XB!R_oji6@LMY*XbYY^nREE*j~h4zn$s%xH!#Jc)QJ*}A`GIRd%;h9 zr{!bY3}7`s!-Rl>klFG##BXj3?n6$#d%tgdy|x2#n(E^$MU+YRI8Z8*;o*gmMTmW) zNeBA8zbMS_rYfLnq23plX&v{tAcWdxm}uM4X`YsI`kDOWMkV48n+n*!I1)$_xpm(5 z4=xq4`0u+dvp~6^gnWa*2ys@Chii-{*4xnU^>W+C1J}C)$2 z3hjmJLk_vT=c&wKY<-88OBROPvWDR9e{2jslXygBHpU(P+Zj!e=8Xqum=a=E)D|n^ zdzhnA*VUZnvAIl@E6jsv$oX0mHsWkYa%Kr^@B(U*bR3M;E`>r_wsHdH0r;%BRp%qdj7OagAT4Hfxv~-q3+3s p@Y&Mlv$OgC*Z)S~&FoF&^{D3SR^jz;@J0~`IjQfGwc;jW{{tW!r?CJ4 literal 0 HcmV?d00001 diff --git a/webui/webui.go b/webui/webui.go new file mode 100644 index 00000000..476807ae --- /dev/null +++ b/webui/webui.go @@ -0,0 +1,282 @@ +package webui + +import ( + _ "embed" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/cooperspencer/gickup/types" + "github.com/goccy/go-yaml" + "github.com/rs/zerolog/log" +) + +// BackupStatus is the result of a single repo-to-destination backup attempt. +type BackupStatus string + +const ( + StatusSuccess BackupStatus = "success" + StatusFailed BackupStatus = "failed" +) + +// StatusFromInt converts the integer convention used by backup() (1=success, 0=failed). +func StatusFromInt(i int) BackupStatus { + if i == 1 { + return StatusSuccess + } + return StatusFailed +} + +// BackupEntry records one repo-to-destination backup attempt. +type BackupEntry struct { + Timestamp time.Time `json:"timestamp"` + RepoName string `json:"repo_name"` + RepoURL string `json:"repo_url"` + Owner string `json:"owner"` + Hoster string `json:"hoster"` + DestType string `json:"dest_type"` + DestAddr string `json:"dest_addr"` + Status BackupStatus `json:"status"` + DurationMs int64 `json:"duration_ms"` +} + +// ConfigInfo describes one loaded configuration block for the UI. +type ConfigInfo struct { + Index int `json:"index"` + Name string `json:"name"` + Sources int `json:"sources"` + Dests int `json:"dests"` + CronSpec string `json:"cron_spec,omitempty"` + NextRun string `json:"next_run,omitempty"` // RFC3339 +} + +// store holds backup entries, config file paths, and the run callback. +type store struct { + mu sync.RWMutex + entries []BackupEntry + configFiles []string + configs []ConfigInfo + runFunc func(int) + running int32 // atomic: 1 = a run is in progress +} + +// Global is the shared store used across the application. +var Global = &store{} + +// SetConfigFiles registers the config file paths with the store. +func (s *store) SetConfigFiles(files []string) { + s.mu.Lock() + defer s.mu.Unlock() + s.configFiles = make([]string, len(files)) + copy(s.configFiles, files) +} + +// SetConfigs registers the list of config summaries shown in the UI. +func (s *store) SetConfigs(infos []ConfigInfo) { + s.mu.Lock() + defer s.mu.Unlock() + s.configs = make([]ConfigInfo, len(infos)) + copy(s.configs, infos) +} + +// SetRunFunc registers the callback that main.go uses to trigger a backup run. +// index == -1 means run all configs; otherwise run the config at that index. +func (s *store) SetRunFunc(fn func(int)) { + s.mu.Lock() + defer s.mu.Unlock() + s.runFunc = fn +} + +// Record appends a backup entry to the store. +func (s *store) Record(e BackupEntry) { + s.mu.Lock() + defer s.mu.Unlock() + s.entries = append(s.entries, e) +} + +// Clear removes all recorded backup entries. +func (s *store) Clear() { + s.mu.Lock() + defer s.mu.Unlock() + s.entries = nil +} + +//go:embed ui/index.html +var indexHTML []byte + +//go:embed ui/logo.png +var logoPNG []byte + +// Serve starts the web UI HTTP server on addr. +func Serve(addr string) { + mux := http.NewServeMux() + mux.HandleFunc("/", handleIndex) + mux.HandleFunc("/logo.png", handleLogo) + mux.HandleFunc("/api/status", handleStatus) + mux.HandleFunc("/api/config", handleConfig) + mux.HandleFunc("/api/configs", handleConfigs) + mux.HandleFunc("/api/run", handleRun) + mux.HandleFunc("/api/running", handleRunning) + log.Info().Str("addr", addr).Msg("Web UI listening") + if err := http.ListenAndServe(addr, mux); err != nil { + log.Error().Err(err).Msg("Web UI server error") + } +} + +func handleIndex(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write(indexHTML) +} + +func handleLogo(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(logoPNG) +} + +func handleStatus(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + Global.Clear() + w.WriteHeader(http.StatusNoContent) + return + } + Global.mu.RLock() + entries := make([]BackupEntry, len(Global.entries)) + copy(entries, Global.entries) + Global.mu.RUnlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(entries) +} + +// handleConfigs returns the list of loaded configuration blocks. +func handleConfigs(w http.ResponseWriter, r *http.Request) { + Global.mu.RLock() + cfgs := make([]ConfigInfo, len(Global.configs)) + copy(cfgs, Global.configs) + Global.mu.RUnlock() + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(cfgs) +} + +// handleRunning reports whether a backup run is currently in progress. +func handleRunning(w http.ResponseWriter, r *http.Request) { + running := atomic.LoadInt32(&Global.running) != 0 + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]bool{"running": running}) +} + +// handleRun triggers an immediate backup run. +// POST /api/run body: {"index": -1} (-1 = all, ≥0 = specific config) +func handleRun(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + if !atomic.CompareAndSwapInt32(&Global.running, 0, 1) { + http.Error(w, "backup already running", http.StatusConflict) + return + } + + var req struct { + Index int `json:"index"` + } + req.Index = -1 + _ = json.NewDecoder(r.Body).Decode(&req) + + Global.mu.RLock() + fn := Global.runFunc + Global.mu.RUnlock() + + if fn == nil { + atomic.StoreInt32(&Global.running, 0) + http.Error(w, "not ready", http.StatusServiceUnavailable) + return + } + + go func() { + defer atomic.StoreInt32(&Global.running, 0) + fn(req.Index) + }() + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]bool{"running": true}) +} + +func handleConfig(w http.ResponseWriter, r *http.Request) { + Global.mu.RLock() + files := make([]string, len(Global.configFiles)) + copy(files, Global.configFiles) + Global.mu.RUnlock() + + if len(files) == 0 { + http.Error(w, "no config files registered", http.StatusNotFound) + return + } + + q := r.URL.Query() + + // List mode: GET /api/config?list=1 + if r.Method == http.MethodGet && q.Get("list") == "1" { + type fileInfo struct { + Index int `json:"index"` + Name string `json:"name"` + Path string `json:"path"` + } + list := make([]fileInfo, len(files)) + for i, f := range files { + list[i] = fileInfo{Index: i, Name: filepath.Base(f), Path: f} + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(list) + return + } + + // Parse file index (default 0). + idx := 0 + if s := q.Get("file"); s != "" { + n, err := strconv.Atoi(s) + if err != nil || n < 0 || n >= len(files) { + http.Error(w, "invalid file index", http.StatusBadRequest) + return + } + idx = n + } + + switch r.Method { + case http.MethodGet: + data, err := os.ReadFile(files[idx]) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + _, _ = w.Write(data) + + case http.MethodPost: + body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + // Validate that the submitted content parses as a known config shape. + var conf types.Conf + if err := yaml.Unmarshal(body, &conf); err != nil { + http.Error(w, "invalid YAML: "+err.Error(), http.StatusBadRequest) + return + } + if err := os.WriteFile(files[idx], body, 0o600); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.WriteHeader(http.StatusOK) + + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} From efab9b659af83ddbd2a4da1be4b4693cbfe810f5 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Thu, 14 May 2026 18:19:58 +0200 Subject: [PATCH 40/44] show correct next cron --- webui/webui.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/webui/webui.go b/webui/webui.go index 476807ae..e011b8d7 100644 --- a/webui/webui.go +++ b/webui/webui.go @@ -159,6 +159,20 @@ func handleConfigs(w http.ResponseWriter, r *http.Request) { cfgs := make([]ConfigInfo, len(Global.configs)) copy(cfgs, Global.configs) Global.mu.RUnlock() + + now := time.Now() + for i := range cfgs { + if cfgs[i].CronSpec == "" { + continue + } + sched, err := types.ParseCronSpec(cfgs[i].CronSpec) + if err != nil { + cfgs[i].NextRun = "" + continue + } + cfgs[i].NextRun = sched.Next(now).Format(time.RFC3339) + } + w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(cfgs) } From f1a2859a01108d25e0bd577abd7eeb37699a4409 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Mon, 22 Jun 2026 09:41:08 +0200 Subject: [PATCH 41/44] fixed css for mobile ui --- webui/ui/index.html | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/webui/ui/index.html b/webui/ui/index.html index 3f8bf226..64bceb77 100644 --- a/webui/ui/index.html +++ b/webui/ui/index.html @@ -71,7 +71,15 @@ flex-shrink: 0; } .logo img { width: 32px; height: 32px; object-fit: contain; } - .nav-tabs { display: flex; align-items: stretch; height: 100%; } + .nav-tabs { + display: flex; + align-items: stretch; + height: 100%; + min-width: 0; + overflow-x: auto; + scrollbar-width: none; + } + .nav-tabs::-webkit-scrollbar { display: none; } .nav-tab { display: flex; align-items: center; @@ -98,9 +106,12 @@ display: flex; align-items: center; gap: 6px; + flex-shrink: 0; transition: color 0.15s, border-color 0.15s; } .theme-toggle:hover { color: var(--text); border-color: var(--text); } + #theme-icon { display: inline-flex; align-items: center; flex-shrink: 0; } + #theme-icon svg { display: block; } /* layout */ main { padding: 28px 24px; max-width: 1280px; margin: 0 auto; } .page { display: none; } @@ -410,6 +421,24 @@ ::-webkit-scrollbar-track { background: transparent; } ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; } ::-webkit-scrollbar-thumb:hover { background: var(--muted); } + + @media (max-width: 560px) { + nav { + padding: 0 12px; + gap: 12px; + } + .logo { gap: 8px; font-size: 15px; } + .logo img { width: 28px; height: 28px; } + .nav-tab { padding: 0 10px; font-size: 13px; } + .theme-toggle { + width: 34px; + height: 34px; + justify-content: center; + padding: 0; + } + #theme-label { display: none; } + main { padding: 20px 12px; } + } @@ -424,7 +453,7 @@ - From 5a8be0b2580d6ed6465ef729351accaf88a8fa20 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Sun, 20 Sep 2026 19:44:14 +0200 Subject: [PATCH 42/44] complete backup reporting and improve dashboard behavior --- main.go | 142 +++++++++++++++++++++++++------------------- webui/ui/index.html | 103 +++++++++++++++++++++----------- webui/webui.go | 12 +++- 3 files changed, 159 insertions(+), 98 deletions(-) diff --git a/main.go b/main.go index 95f6e630..02d2bdde 100644 --- a/main.go +++ b/main.go @@ -478,74 +478,86 @@ func backup(repos []types.Repo, conf *types.Conf) { Msgf("%s %s to %s", logOp, types.Blue(r.Name), d.Url) if !cli.Dry { - tempname := fmt.Sprintf("webdav-%x", repotime) - tempdir, err := os.MkdirTemp(os.TempDir(), tempname) - if err != nil { - log.Error(). - Str("stage", "tempclone"). - Str("url", r.URL). - Msg(err.Error()) - continue - } - - if d.Structured { - r.Name = path.Join(r.Hoster, r.Owner, r.Name) - } - - if d.DateCreateDir { - r.Name = currentDateDir + r.Name - } - - defer os.RemoveAll(tempdir) - tempClonePath := path.Join(tempdir, r.Name) - _, err = local.TempCloneBare(r, tempClonePath) - if err != nil { - switch { - case errors.Is(err, git.NoErrAlreadyUpToDate): - log.Info(). - Str("stage", "webdav"). - Msg(err.Error()) - case errors.Is(err, transport.ErrEmptyRemoteRepository): - log.Warn(). - Str("repo", r.Name). - Msgf("%s - Skipping backup", err.Error()) - continue - default: + func() { + defer func() { + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, RepoName: r.Name, RepoURL: r.URL, + Owner: r.Owner, Hoster: r.Hoster, DestType: "webdav", + DestAddr: d.Url, Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) + }() + tempname := fmt.Sprintf("webdav-%x", repotime) + tempdir, err := os.MkdirTemp(os.TempDir(), tempname) + if err != nil { log.Error(). Str("stage", "tempclone"). - Str("git", "clone"). + Str("url", r.URL). Msg(err.Error()) - os.RemoveAll(tempdir) - continue + return } - } - if d.Zip { - log.Info(). - Msgf("zipping %s", types.Green(r.Name)) - err := zip.Zip(tempClonePath, []string{tempClonePath}) + if d.Structured { + r.Name = path.Join(r.Hoster, r.Owner, r.Name) + } + + if d.DateCreateDir { + r.Name = currentDateDir + r.Name + } + + defer os.RemoveAll(tempdir) + tempClonePath := path.Join(tempdir, r.Name) + _, err = local.TempCloneBare(r, tempClonePath) if err != nil { - log.Error(). - Str("stage", "zip"). - Str("repo", r.Name). - Msg(err.Error()) - log.Error().Msgf("Skipping backup of %s due to error while zipping", r.Name) - continue + switch { + case errors.Is(err, git.NoErrAlreadyUpToDate): + log.Info(). + Str("stage", "webdav"). + Msg(err.Error()) + case errors.Is(err, transport.ErrEmptyRemoteRepository): + log.Warn(). + Str("repo", r.Name). + Msgf("%s - Skipping backup", err.Error()) + return + default: + log.Error(). + Str("stage", "tempclone"). + Str("git", "clone"). + Msg(err.Error()) + os.RemoveAll(tempdir) + return + } } - } - err = webdav.UploadDirToWebDAV(tempdir, d) - if err != nil { - log.Error().Str("stage", "webdav").Str("url", d.Url).Msg(err.Error()) - } - err = webdav.DeleteObjectsNotInRepo(tempdir, r.Name, d) - if err != nil { - log.Error().Str("stage", "webdav").Str("url", d.Url).Msg(err.Error()) - } - prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "webdav", d.Url).Set(time.Since(repotime).Seconds()) - status = 1 - prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "webdav", d.Url).Set(float64(status)) - prometheus.DestinationBackupsComplete.WithLabelValues("webdav").Inc() + if d.Zip { + log.Info(). + Msgf("zipping %s", types.Green(r.Name)) + err := zip.Zip(tempClonePath, []string{tempClonePath}) + if err != nil { + log.Error(). + Str("stage", "zip"). + Str("repo", r.Name). + Msg(err.Error()) + log.Error().Msgf("Skipping backup of %s due to error while zipping", r.Name) + return + } + } + err = webdav.UploadDirToWebDAV(tempdir, d) + if err != nil { + log.Error().Str("stage", "webdav").Str("url", d.Url).Msg(err.Error()) + return + } + err = webdav.DeleteObjectsNotInRepo(tempdir, r.Name, d) + if err != nil { + log.Error().Str("stage", "webdav").Str("url", d.Url).Msg(err.Error()) + return + } + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "webdav", d.Url).Set(time.Since(repotime).Seconds()) + status = 1 + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "webdav", d.Url).Set(float64(status)) + prometheus.DestinationBackupsComplete.WithLabelValues("webdav").Inc() + }() } } @@ -1117,6 +1129,14 @@ func backup(repos []types.Repo, conf *types.Conf) { if !cli.Dry { func() { + defer func() { + webui.Global.Record(webui.BackupEntry{ + Timestamp: repotime, RepoName: r.Name, RepoURL: r.URL, + Owner: r.Owner, Hoster: r.Hoster, DestType: "radicle", + DestAddr: radhome, Status: webui.StatusFromInt(status), + DurationMs: time.Since(repotime).Milliseconds(), + }) + }() tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("radicle-%x", repotime)) if err != nil { log.Error(). @@ -1172,6 +1192,8 @@ func backup(repos []types.Repo, conf *types.Conf) { } func runBackup(conf *types.Conf, num int) { + webui.Global.BeginRun() + defer webui.Global.EndRun() log.Info().Msg("Backup run starting") numstring := strconv.Itoa(num) diff --git a/webui/ui/index.html b/webui/ui/index.html index 64bceb77..e75ed173 100644 --- a/webui/ui/index.html +++ b/webui/ui/index.html @@ -468,7 +468,7 @@
Successful
—
Failed
—
Unique Repos
—
-
Sources
—
+
Source Hosts
—
@@ -512,18 +512,19 @@

Backup History

+

Counts and filters reflect recorded backups since startup or the last history clear.

- +
@@ -548,6 +549,7 @@

Configuration

+

Edit configuration as YAML. Reload reads the file from disk and discards edits. Saved changes are picked up automatically; without a schedule, reloading the configuration starts a backup.

+
@@ -549,6 +554,22 @@

Configuration

+

+
+ Configuration example and supported providers +

Sources: GitHub, GitLab, Gitea/Forgejo, Gogs, Bitbucket, OneDev, SourceHut, Opengist, or any Git URL.

+

Destinations: local, GitHub, GitLab, Gitea/Forgejo, Gogs, OneDev, SourceHut, S3, Azure Blob, WebDAV, or Radicle.

+

Copy this example into the editor and replace the repository URL and backup path. Without a cron schedule, saving starts a backup after the configuration is loaded.

+
source:
+  any:
+    - url: https://github.com/OWNER/REPOSITORY.git
+destination:
+  local:
+    - path: /backups
+      bare: true
+# Optional daily schedule (UTC unless TZ is set):
+# cron: "0 0 * * *"
+

Edit configuration as YAML. Reload reads the file from disk and discards edits. Saved changes are picked up automatically; without a schedule, reloading the configuration starts a backup.

@@ -655,6 +676,9 @@

Configuration

let allEntries = []; let refreshInterval = null; let runPollInterval = null; + let hasConfigs = false; + let backupRunning = false; + let setupShown = false; // Navigation function showPage(name) { @@ -671,6 +695,13 @@

Configuration

const resp = await fetch('/api/configs'); if (!resp.ok) return; const cfgs = await resp.json(); + hasConfigs = cfgs.length > 0; + document.getElementById('setup-notice').style.display = hasConfigs ? 'none' : ''; + showRunning(backupRunning); + if (!hasConfigs && !setupShown) { + setupShown = true; + showPage('config'); + } const body = document.getElementById('run-body'); // Keep the "Run All" button, remove old config buttons body.querySelectorAll('.btn-config').forEach(b => b.remove()); @@ -741,8 +772,9 @@

Configuration

} function showRunning(on) { + backupRunning = on; document.getElementById('run-status').style.display = on ? 'inline-flex' : 'none'; - document.querySelectorAll('#run-body .btn').forEach(b => b.disabled = on); + document.querySelectorAll('#run-body .btn').forEach(b => b.disabled = on || !hasConfigs); } async function pollRunning() { @@ -750,7 +782,7 @@

Configuration

const resp = await fetch('/api/running'); if (!resp.ok) return; const { running } = await resp.json(); - const wasRunning = document.getElementById('btn-run-all').disabled; + const wasRunning = backupRunning; showRunning(running); if (wasRunning && !running) loadStatus(); } catch (e) { console.error('pollRunning:', e); } @@ -900,6 +932,7 @@

Configuration

// config let activeConfigFile = 0; + let configFiles = []; let configLoaded = false; let savedConfigContent = ''; let configLoading = false; @@ -922,6 +955,7 @@

Configuration

const resp = await fetch('/api/config?list=1'); if (!resp.ok) throw new Error(await resp.text()); const files = await resp.json(); + configFiles = files; if (files.length <= 1) { tabsEl.style.display = 'none'; @@ -954,6 +988,8 @@

Configuration

if (configLoading || configSaving) return; if (configDirty() && !confirm('Discard unsaved configuration changes and reload from disk?')) return; configLoading = true; + const file = configFiles.find(f => f.index === activeConfigFile); + document.getElementById('config-path').textContent = file ? 'Configuration file: ' + file.path : ''; const el = document.getElementById('config-content'); const st = document.getElementById('save-status'); el.disabled = true; @@ -964,6 +1000,7 @@

Configuration

const resp = await fetch(`/api/config?file=${activeConfigFile}`); if (!resp.ok) throw new Error(await resp.text()); el.value = await resp.text(); + el.placeholder = "Paste your configuration YAML here, or copy the example above."; savedConfigContent = el.value; configLoaded = true; } catch (e) { @@ -994,7 +1031,8 @@

Configuration

if (resp.ok) { if (fileIndex !== activeConfigFile) return; savedConfigContent = content; - st.textContent = '✓ Saved to disk'; + st.textContent = '✓ Saved — Gickup will reload automatically'; + loadConfigs(); st.className = 'save-status ok'; } else { const msg = await resp.text(); @@ -1028,7 +1066,7 @@

Configuration

loadStatus(); loadConfigs(); refreshInterval = setInterval(loadStatus, 30000); - setInterval(loadConfigs, 60000); // refresh cron next-run times + setInterval(loadConfigs, 5000); // detect newly saved configuration and refresh schedules // Keep checking so scheduled and startup runs also appear in the UI. pollRunning(); diff --git a/webui/ui/index.test.cjs b/webui/ui/index.test.cjs new file mode 100644 index 00000000..ea2ad54a --- /dev/null +++ b/webui/ui/index.test.cjs @@ -0,0 +1,94 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const vm = require('node:vm'); + +function setup() { + const elements = new Map(); + const element = id => { + if (!elements.has(id)) elements.set(id, { + value: '', innerHTML: '', textContent: '', style: {}, disabled: false, + classList: { add() {}, remove() {}, toggle() {} }, + querySelectorAll: () => [], + }); + return elements.get(id); + }; + const listeners = {}; + const context = vm.createContext({ + document: { + getElementById: element, querySelectorAll: () => [], + documentElement: { setAttribute() {} }, + }, + localStorage: { getItem: () => 'dark', setItem() {} }, + window: { matchMedia: () => ({ matches: true }), addEventListener: (name, fn) => listeners[name] = fn }, + confirm: () => false, console, setInterval: () => 1, + fetch: async () => { throw new Error('Unexpected request'); }, + }); + const html = readFileSync(__dirname + '/index.html', 'utf8'); + const script = html.match(/