commit d9f9d86c82f962e978a5cec6444969b98ed68a54 · hailey.at/myaur

+67 -1

README.md

···

       1
       1
        
       # myaur

     

       2
       2
        
       

     

       3
       3
       -
       A simple AUR mirror.

     

       3
       3
       +
       A simple, self-hosted AUR (Arch User Repository) mirror written in Go. Provides both RPC API endpoints and git protocol access to AUR packages.

     

       4
       4
       +
       

     

       5
       5
       +
       myaur takes advantage of the [official AUR mirror](https://github.com/archlinux/aur.git) on GitHub. You may use any mirror that you wish, however, note that it must have the same format as the official repo, in that each individual package should be a branch within the repo.

     

       6
       6
       +
       

     

       7
       7
       +
       ## Installation

     

       8
       8
       +
       

     

       9
       9
       +
       ### Using Docker Compose

     

       10
       10
       +
       

     

       11
       11
       +
       The easiest way to start the mirror is to use `docker compose up -d`. This will start both the myaur service and set up a Caddy reverse proxy.

     

       12
       12
       +
       

     

       13
       13
       +
       ```bash

     

       14
       14
       +
       docker compose up -d

     

       15
       15
       +
       ```

     

       16
       16
       +
       

     

       17
       17
       +
       If you wish to use your own domain, modify the `Caddyfile` and change `:443` to your domain.

     

       18
       18
       +
       

     

       19
       19
       +
       ### Building from Source

     

       20
       20
       +
       

     

       21
       21
       +
       Requirements:

     

       22
       22
       +
       - Go 1.25.3 or later

     

       23
       23
       +
       - Git

     

       24
       24
       +
       

     

       25
       25
       +
       ```bash

     

       26
       26
       +
       go build -o myaur ./cmd/myaur

     

       27
       27
       +
       ```

     

       28
       28
       +
       

     

       29
       29
       +
       ## Usage

     

       30
       30
       +
       

     

       31
       31
       +
       ### Populate Database

     

       32
       32
       +
       

     

       33
       33
       +
       If you wish to clone the mirror repo and populate the database, you can do so without actually serving the mirror API.

     

       34
       34
       +
       

     

       35
       35
       +
       ```bash

     

       36
       36
       +
       ./myaur populate \

     

       37
       37
       +
         --database-path ./myaur.db \

     

       38
       38
       +
         --repo-path ./aur-mirror \

     

       39
       39
       +
         --concurrency 10

     

       40
       40
       +
       ```

     

       41
       41
       +
       

     

       42
       42
       +
       Options:

     

       43
       43
       +
       - `--database-path`: Path to SQLite database file (default: `./myaur.db`)

     

       44
       44
       +
       - `--repo-path`: Path to clone/update AUR git mirror (default: `./aur-mirror`)

     

       45
       45
       +
       - `--remote-repo-url`: Remote AUR repository URL (default: `https://github.com/archlinux/aur.git`)

     

       46
       46
       +
       - `--concurrency`: Number of worker threads for parsing (default: `10`)

     

       47
       47
       +
       - `--debug`: Enable debug logging

     

       48
       48
       +
       

     

       49
       49
       +
       ### Serve

     

       50
       50
       +
       

     

       51
       51
       +
       To serve the API:

     

       52
       52
       +
       

     

       53
       53
       +
       ```bash

     

       54
       54
       +
       ./myaur serve \

     

       55
       55
       +
         --listen-addr :8080 \

     

       56
       56
       +
         --database-path ./myaur.db \

     

       57
       57
       +
         --repo-path ./aur-mirror \

     

       58
       58
       +
         --concurrency 10

     

       59
       59
       +
       ```

     

       60
       60
       +
       

     

       61
       61
       +
       Options:

     

       62
       62
       +
       - `--listen-addr`: HTTP server listen address (default: `:8080`)

     

       63
       63
       +
       - `--database-path`: Path to SQLite database file (default: `./myaur.db`)

     

       64
       64
       +
       - `--repo-path`: Path to AUR git mirror (default: `./aur-mirror`)

     

       65
       65
       +
       - `--remote-repo-url`: Remote AUR repository URL (default: `https://github.com/archlinux/aur.git`)

     

       66
       66
       +
       - `--concurrency`: Number of worker threads for parsing (default: `10`)

     

       67
       67
       +
       - `--auto-update`: Whether or not to automtically fetch updates from the remote repo (default: `true`)

     

       68
       68
       +
       - `--update-interval`: Time between automatic fetches (default: `1h`)

     

       69
       69
       +
       - `--debug`: Enable debug logging

+25 -7

cmd/myaur/main.go

···

       5
       5
        
       	"fmt"

     

       6
       6
        
       	"log"

     

       7
       7
        
       	"os"

     

       8
       8
       +
       	"time"

     

       8
       9
        
       

     

       9
       10
        
       	"github.com/haileyok/myaur/myaur/gitrepo"

     

       10
       11
        
       	"github.com/haileyok/myaur/myaur/populate"

     
···

       43
       44
        
       					&cli.IntFlag{

     

       44
       45
        
       						Name:  "concurrency",

     

       45
       46
        
       						Usage: "worker concurrency for parsing and adding packages to database",

     

       46
       46
       -
       						Value: 10, // TODO: is this a good default

     

       47
       47
       +
       						Value: 10,

     

       47
       48
        
       					},

     

       48
       49
        
       				},

     

       49
       50
        
       				Action: func(cmd *cli.Context) error {

     
···

       99
       100
        
       						Name:  "debug",

     

       100
       101
        
       						Usage: "flag to enable debug logs",

     

       101
       102
        
       					},

     

       103
       103
       +
       					&cli.IntFlag{

     

       104
       104
       +
       						Name:  "concurrency",

     

       105
       105
       +
       						Usage: "worker concurrency for parsing and adding packages to database",

     

       106
       106
       +
       						Value: 10,

     

       107
       107
       +
       					},

     

       108
       108
       +
       					&cli.BoolFlag{

     

       109
       109
       +
       						Name:  "auto-update",

     

       110
       110
       +
       						Usage: "automatically pull updates from the remote repo at the set interval",

     

       111
       111
       +
       						Value: true,

     

       112
       112
       +
       					},

     

       113
       113
       +
       					&cli.DurationFlag{

     

       114
       114
       +
       						Name:  "update-interval",

     

       115
       115
       +
       						Usage: "the interval at which updates will be fetched. note that this should likely be at most one hour.",

     

       116
       116
       +
       						Value: time.Hour,

     

       117
       117
       +
       					},

     

       102
       118
        
       				},

     

       103
       119
        
       				Action: func(cmd *cli.Context) error {

     

       104
       120
        
       					ctx := context.Background()

     

       105
       121
        
       

     

       106
       122
        
       					s, err := server.New(&server.Args{

     

       107
       107
       -
       						Addr:          cmd.String("listen-addr"),

     

       108
       108
       -
       						MetricsAddr:   cmd.String("metrics-listen-addr"),

     

       109
       109
       -
       						DatabasePath:  cmd.String("database-path"),

     

       110
       110
       -
       						RemoteRepoUrl: cmd.String("remote-repo-url"),

     

       111
       111
       -
       						RepoPath:      cmd.String("repo-path"),

     

       112
       112
       -
       						Debug:         cmd.Bool("debug"),

     

       123
       123
       +
       						Addr:           cmd.String("listen-addr"),

     

       124
       124
       +
       						DatabasePath:   cmd.String("database-path"),

     

       125
       125
       +
       						RemoteRepoUrl:  cmd.String("remote-repo-url"),

     

       126
       126
       +
       						RepoPath:       cmd.String("repo-path"),

     

       127
       127
       +
       						Concurrency:    cmd.Int("concurrency"),

     

       128
       128
       +
       						AutoUpdate:     cmd.Bool("auto-update"),

     

       129
       129
       +
       						UpdateInterval: cmd.Duration("update-interval"),

     

       130
       130
       +
       						Debug:          cmd.Bool("debug"),

     

       113
       131
        
       					})

     

       114
       132
        
       					if err != nil {

     

       115
       133
        
       						return fmt.Errorf("failed to create new myaur server: %w", err)

+61 -67

myaur/server/server.go

···

       21
       21
        
       )

     

       22
       22
        
       

     

       23
       23
        
       type Server struct {

     

       24
       24
       -
       	logger        *slog.Logger

     

       25
       25
       -
       	echo          *echo.Echo

     

       26
       26
       -
       	httpd         *http.Server

     

       27
       27
       -
       	metricsHttpd  *http.Server

     

       28
       28
       -
       	db            *database.Database

     

       29
       29
       -
       	populator     *populate.Populate

     

       30
       30
       -
       	remoteRepoUrl string

     

       31
       31
       -
       	repoPath      string

     

       24
       24
       +
       	logger         *slog.Logger

     

       25
       25
       +
       	echo           *echo.Echo

     

       26
       26
       +
       	httpd          *http.Server

     

       27
       27
       +
       	db             *database.Database

     

       28
       28
       +
       	populator      *populate.Populate

     

       29
       29
       +
       	remoteRepoUrl  string

     

       30
       30
       +
       	repoPath       string

     

       31
       31
       +
       	autoUpdate     bool

     

       32
       32
       +
       	updateInterval time.Duration

     

       32
       33
        
       }

     

       33
       34
        
       

     

       34
       35
        
       type Args struct {

     

       35
       35
       -
       	Addr          string

     

       36
       36
       -
       	MetricsAddr   string

     

       37
       37
       -
       	DatabasePath  string

     

       38
       38
       -
       	RemoteRepoUrl string

     

       39
       39
       -
       	RepoPath      string

     

       40
       40
       -
       	Debug         bool

     

       36
       36
       +
       	Addr           string

     

       37
       37
       +
       	DatabasePath   string

     

       38
       38
       +
       	RemoteRepoUrl  string

     

       39
       39
       +
       	RepoPath       string

     

       40
       40
       +
       	Concurrency    int

     

       41
       41
       +
       	AutoUpdate     bool

     

       42
       42
       +
       	UpdateInterval time.Duration

     

       43
       43
       +
       	Debug          bool

     

       41
       44
        
       }

     

       42
       45
        
       

     

       43
       46
        
       func New(args *Args) (*Server, error) {

     
···

       64
       67
        
       		Handler: e,

     

       65
       68
        
       	}

     

       66
       69
        
       

     

       67
       67
       -
       	metricsHttpd := http.Server{

     

       68
       68
       -
       		Addr: args.MetricsAddr,

     

       69
       69
       -
       	}

     

       70
       70
       -
       

     

       71
       70
        
       	db, err := database.New(&database.Args{

     

       72
       71
        
       		DatabasePath: args.DatabasePath,

     

       73
       72
        
       		Debug:        args.Debug,

     
···

       81
       80
        
       		RepoPath:      args.RepoPath,

     

       82
       81
        
       		RemoteRepoUrl: args.RemoteRepoUrl,

     

       83
       82
        
       		Debug:         args.Debug,

     

       84
       84
       -
       		Concurrency:   20, // TODO: make an env-var for this

     

       83
       83
       +
       		Concurrency:   args.Concurrency,

     

       85
       84
        
       	})

     

       86
       85
        
       

     

       87
       86
        
       	s := Server{

     

       88
       88
       -
       		echo:          e,

     

       89
       89
       -
       		httpd:         &httpd,

     

       90
       90
       -
       		metricsHttpd:  &metricsHttpd,

     

       91
       91
       -
       		db:            db,

     

       92
       92
       -
       		populator:     populator,

     

       93
       93
       -
       		logger:        logger,

     

       94
       94
       -
       		remoteRepoUrl: args.RemoteRepoUrl,

     

       95
       95
       -
       		repoPath:      args.RepoPath,

     

       87
       87
       +
       		echo:           e,

     

       88
       88
       +
       		httpd:          &httpd,

     

       89
       89
       +
       		db:             db,

     

       90
       90
       +
       		populator:      populator,

     

       91
       91
       +
       		logger:         logger,

     

       92
       92
       +
       		remoteRepoUrl:  args.RemoteRepoUrl,

     

       93
       93
       +
       		repoPath:       args.RepoPath,

     

       94
       94
       +
       		autoUpdate:     args.AutoUpdate,

     

       95
       95
       +
       		updateInterval: args.UpdateInterval,

     

       96
       96
        
       	}

     

       97
       97
        
       

     

       98
       98
        
       	return &s, nil

     

       99
       99
        
       }

     

       100
       100
        
       

     

       101
       101
        
       func (s *Server) Serve(ctx context.Context) error {

     

       102
       102
       -
       	go func() {

     

       103
       103
       -
       		logger := s.logger.With("component", "metrics-httpd")

     

       104
       104
       -
       

     

       105
       105
       -
       		go func() {

     

       106
       106
       -
       			if err := s.metricsHttpd.ListenAndServe(); err != http.ErrServerClosed {

     

       107
       107
       -
       				logger.Error("error listening", "err", err)

     

       108
       108
       -
       			}

     

       109
       109
       -
       		}()

     

       110
       110
       -
       

     

       111
       111
       -
       		logger.Info("myaur metrics server listening", "addr", s.metricsHttpd.Addr)

     

       112
       112
       -
       	}()

     

       113
       113
       -
       

     

       114
       102
        
       	shutdownTicker := make(chan struct{})

     

       115
       103
        
       	tickerShutdown := make(chan struct{})

     

       116
       116
       -
       	go func() {

     

       117
       117
       -
       		logger := s.logger.With("component", "update-routine")

     

       118
       118
       -
       

     

       119
       119
       -
       		ticker := time.NewTicker(1 * time.Hour)

     

       120
       120
       -
       

     

       104
       104
       +
       	if s.autoUpdate {

     

       121
       105
        
       		go func() {

     

       122
       122
       -
       			logger.Info("performing initial database population")

     

       106
       106
       +
       			logger := s.logger.With("component", "update-routine")

     

       123
       107
        
       

     

       124
       124
       -
       			if err := s.populator.Run(ctx); err != nil {

     

       125
       125
       -
       				logger.Info("error populating", "err", err)

     

       126
       126
       -
       			}

     

       108
       108
       +
       			ticker := time.NewTicker(s.updateInterval)

     

       127
       109
        
       

     

       128
       128
       -
       			for range ticker.C {

     

       110
       110
       +
       			go func() {

     

       111
       111
       +
       				logger.Info("performing initial database population")

     

       112
       112
       +
       

     

       129
       113
        
       				if err := s.populator.Run(ctx); err != nil {

     

       130
       114
        
       					logger.Info("error populating", "err", err)

     

       131
       115
        
       				}

     

       132
       132
       -
       			}

     

       133
       116
        
       

     

       134
       134
       -
       			close(tickerShutdown)

     

       135
       135
       -
       		}()

     

       117
       117
       +
       				for range ticker.C {

     

       118
       118
       +
       					if err := s.populator.Run(ctx); err != nil {

     

       119
       119
       +
       						logger.Info("error populating", "err", err)

     

       120
       120
       +
       					}

     

       121
       121
       +
       				}

     

       136
       122
        
       

     

       137
       137
       -
       		<-shutdownTicker

     

       123
       123
       +
       				close(tickerShutdown)

     

       124
       124
       +
       			}()

     

       138
       125
        
       

     

       139
       139
       -
       		ticker.Stop()

     

       140
       140
       -
       	}()

     

       126
       126
       +
       			<-shutdownTicker

     

       127
       127
       +
       

     

       128
       128
       +
       			ticker.Stop()

     

       129
       129
       +
       		}()

     

       130
       130
       +
       	}

     

       141
       131
        
       

     

       142
       132
        
       	shutdownEcho := make(chan struct{})

     

       143
       133
        
       	echoShutdown := make(chan struct{})

     
···

       191
       181
        
       		// echo should have already been closed

     

       192
       182
        
       	}

     

       193
       183
        
       

     

       194
       194
       -
       	close(shutdownTicker)

     

       184
       184
       +
       	if s.autoUpdate {

     

       185
       185
       +
       		close(shutdownTicker)

     

       186
       186
       +
       	}

     

       195
       187
        
       

     

       196
       188
        
       	s.logger.Info("send ctrl+c to forcefully shutdown without waiting for routines to finish")

     

       197
       189
        
       

     
···

       211
       203
        
       		}

     

       212
       204
        
       	})

     

       213
       205
        
       

     

       214
       214
       -
       	wg.Go(func() {

     

       215
       215
       -
       		s.logger.Info("waiting up to 60 seconds for ticker to shut down")

     

       216
       216
       -
       		select {

     

       217
       217
       -
       		case <-tickerShutdown:

     

       218
       218
       -
       			s.logger.Info("ticker shutdown gracefully")

     

       219
       219
       -
       		case <-time.After(60 * time.Second):

     

       220
       220
       -
       			s.logger.Warn("waited 60 seconds for ticker to shut down. forcefully exiting.")

     

       221
       221
       -
       		case <-forceShutdownSignals:

     

       222
       222
       -
       			s.logger.Warn("received forceful shutdown signal before ticker shut down")

     

       223
       223
       -
       		}

     

       224
       224
       -
       	})

     

       206
       206
       +
       	if s.autoUpdate {

     

       207
       207
       +
       		wg.Go(func() {

     

       208
       208
       +
       			s.logger.Info("waiting up to 60 seconds for ticker to shut down")

     

       209
       209
       +
       			select {

     

       210
       210
       +
       			case <-tickerShutdown:

     

       211
       211
       +
       				s.logger.Info("ticker shutdown gracefully")

     

       212
       212
       +
       			case <-time.After(60 * time.Second):

     

       213
       213
       +
       				s.logger.Warn("waited 60 seconds for ticker to shut down. forcefully exiting.")

     

       214
       214
       +
       			case <-forceShutdownSignals:

     

       215
       215
       +
       				s.logger.Warn("received forceful shutdown signal before ticker shut down")

     

       216
       216
       +
       			}

     

       217
       217
       +
       		})

     

       218
       218
       +
       	}

     

       225
       219
        
       

     

       226
       220
        
       	s.logger.Info("waiting for routines to finish")

     

       227
       221
        
       	wg.Wait()