From 733cebfd819aaed40622ea19c30869d2e0ec6398 Mon Sep 17 00:00:00 2001 From: Maxim Slipenko Date: Sat, 14 Dec 2024 08:35:45 +0000 Subject: [PATCH] feat: add database (#7) Reviewed-on: https://code.alt-gnome.ru/aides-infra/aides-repo-api/pulls/7 Co-authored-by: Maxim Slipenko Co-committed-by: Maxim Slipenko --- .dockerignore | 4 + .gitignore | 4 +- Dockerfile | 14 +- cmd/aides-repo-api/main.go | 13 +- go.mod | 22 ++- go.sum | 43 +++++ internal/app/app.go | 36 ++++ internal/config/config.go | 6 + .../controllers/taskcontroller/controller.go | 88 +--------- internal/controllers/taskcontroller/create.go | 56 +++++++ internal/controllers/taskcontroller/upload.go | 75 +++++++++ internal/models/db.go | 77 +++++++++ internal/models/models.go | 12 -- internal/router/router.go | 41 ++++- internal/services/reposervice/service.go | 156 ++++++++++++++++++ internal/services/taskservice/create.go | 31 ++++ internal/services/taskservice/service.go | 138 +++++++++------- internal/services/taskservice/upload.go | 126 ++++++++++++++ 18 files changed, 769 insertions(+), 173 deletions(-) create mode 100644 .dockerignore create mode 100644 internal/app/app.go create mode 100644 internal/controllers/taskcontroller/create.go create mode 100644 internal/controllers/taskcontroller/upload.go create mode 100644 internal/models/db.go delete mode 100644 internal/models/models.go create mode 100644 internal/services/reposervice/service.go create mode 100644 internal/services/taskservice/create.go create mode 100644 internal/services/taskservice/upload.go diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cb1343b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git +app.db +uploads +Dockerfile \ No newline at end of file diff --git a/.gitignore b/.gitignore index c7348ad..a450265 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,6 @@ fabric.properties .idea/**/azureSettings.xml # Игнорируем папку uploads -uploads \ No newline at end of file +uploads + +app.db \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 545fbf9..0e2f0ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,21 @@ # Первый этап: сборка приложения FROM golang:latest AS builder -WORKDIR /rest-app +WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN go build -o alt-storage +RUN go build -o aides-repo-api ./cmd/aides-repo-api/main.go # Второй этап: создание финального образа FROM registry.altlinux.org/alt/alt:sisyphus -COPY --from=builder /rest-app/alt-storage /bin/main -ENTRYPOINT ["/bin/main"] \ No newline at end of file +RUN \ + apt-get update && \ + apt-get install apt-repo-tools && \ + rm -f /var/lib/apt/lists/*.* && \ + rm -f /var/cache/apt/*.bin + +COPY --from=builder /app/aides-repo-api /bin/aides-repo-api +ENTRYPOINT ["/bin/aides-repo-api"] \ No newline at end of file diff --git a/cmd/aides-repo-api/main.go b/cmd/aides-repo-api/main.go index 8ff30a0..bd476bb 100644 --- a/cmd/aides-repo-api/main.go +++ b/cmd/aides-repo-api/main.go @@ -4,16 +4,19 @@ import ( "log" "net/http" - "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/router" ) func main() { - config := config.New() + app, err := app.New() + if err != nil { + panic(err) + } // Конфигурация сервера - router := router.New(config).Setup() + router := router.New(app).Setup() - log.Printf("Сервер запущен на порту: %s", config.Port) - http.ListenAndServe(":"+config.Port, router) + log.Printf("Сервер запущен на порту: %s", app.Config.Port) + http.ListenAndServe(":"+app.Config.Port, router) } diff --git a/go.mod b/go.mod index fc13160..e9427b5 100644 --- a/go.mod +++ b/go.mod @@ -8,4 +8,24 @@ require ( github.com/go-chi/render v1.0.3 ) -require github.com/ajg/form v1.5.1 // indirect +require ( + github.com/ajg/form v1.5.1 // indirect + github.com/go-co-op/gocron/v2 v2.13.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/jonboulle/clockwork v0.4.0 // indirect + github.com/mattn/go-sqlite3 v1.14.24 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + golang.org/x/crypto v0.31.0 // indirect + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/text v0.21.0 // indirect + gorm.io/driver/postgres v1.5.11 // indirect + gorm.io/driver/sqlite v1.5.7 // indirect + gorm.io/gorm v1.25.12 // indirect +) diff --git a/go.sum b/go.sum index 55110ff..a7284fa 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,50 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg= github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= +github.com/go-co-op/gocron/v2 v2.13.0 h1:iGU/RoZvf4GF5hIZUkDSFvvajk9K3W4YgocarBol/ME= +github.com/go-co-op/gocron/v2 v2.13.0/go.mod h1:ZF70ZwEqz0OO4RBXE1sNxnANy/zvwLcattWEFsqpKig= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.11 h1:ubBVAfbKEUld/twyKZ0IYn9rSQh448EdelLYk9Mv314= +gorm.io/driver/postgres v1.5.11/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= +gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I= +gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= +gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 0000000..0b8d304 --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,36 @@ +package app + +import ( + "fmt" + + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +type App struct { + Db *gorm.DB + Config *config.Config +} + +func New() (*App, error) { + cfg := config.New() + dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", + cfg.DBHost, cfg.DBPort, cfg.DBUser, cfg.DBPassword, cfg.DBName) + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) + if err != nil { + return nil, err + } + + db.AutoMigrate(&models.Task{}, &models.GitRepoAltRepoTask{}, &models.RPMFile{}) + + db.FirstOrCreate(&models.ALTRepo{ + Name: "Sisyphus", + }) + + return &App{ + Config: cfg, + Db: db, + }, nil +} diff --git a/internal/config/config.go b/internal/config/config.go index 36a30ab..08ee12d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,12 @@ type Config struct { UploadDir string `env:"UPLOAD_DIR" envDefault:"./uploads"` Port string `env:"PORT" envDefault:"8080"` MaxSizeUpload int64 `env:"MAX_SIZE_UPLOAD" envDefault:"104857600"` //100 MB + + DBHost string `env:"DB_HOST" envDefault:"localhost"` + DBPort string `env:"DB_PORT" envDefault:"5432"` + DBUser string `env:"DB_USER" envDefault:"user"` + DBPassword string `env:"DB_PASSWORD" envDefault:"password"` + DBName string `env:"DB_NAME" envDefault:"app_db"` } func New() *Config { diff --git a/internal/controllers/taskcontroller/controller.go b/internal/controllers/taskcontroller/controller.go index 24702e4..46efa1b 100644 --- a/internal/controllers/taskcontroller/controller.go +++ b/internal/controllers/taskcontroller/controller.go @@ -1,98 +1,18 @@ package taskcontroller import ( - "net/http" - - "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/common/errors" - "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/taskservice" - "github.com/go-chi/chi/v5" - "github.com/go-chi/render" ) -type TaskUploadResponse struct { - TaskID string `json:"taskID"` - Repo string `json:"repo"` - StatusText string `json:"status"` -} - -func (rd *TaskUploadResponse) Render(w http.ResponseWriter, r *http.Request) error { - render.Status(r, http.StatusOK) - return nil -} - type TaskController struct { - config *config.Config + app *app.App taskService *taskservice.Service } -func New(cfg *config.Config, taskService *taskservice.Service) *TaskController { +func New(app *app.App, taskService *taskservice.Service) *TaskController { return &TaskController{ - config: cfg, + app: app, taskService: taskService, } } - -func (c *TaskController) Upload(w http.ResponseWriter, r *http.Request) { - taskID := chi.URLParam(r, "taskID") - if taskID == "" { - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusBadRequest, - StatusText: "taskID is required", - }) - return - } - - err := r.ParseMultipartForm(10240 << 20) - if err != nil { - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusBadRequest, - StatusText: "Bad Request", - }) - return - } - - repo := r.FormValue("repo") - if repo == "" { - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusBadRequest, - StatusText: "Missing required repo field", - }) - return - } - - files := r.MultipartForm.File["files"] - for _, fileHeader := range files { - if fileHeader.Size > (1024 << 20) { // Limit each file size to 10MB - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusBadRequest, - StatusText: "File too large", - }) - return - } - } - - err = c.taskService.Upload(&taskservice.TaskUploadInput{ - TaskID: taskID, - Repo: repo, - Files: files, - }) - if err != nil { - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusInternalServerError, - StatusText: "Internal Server Error", - Err: err, - }) - } - - response := TaskUploadResponse{ - TaskID: taskID, - Repo: repo, - StatusText: "Success!", - } - - if err := render.Render(w, r, &response); err != nil { - render.Render(w, r, errors.ErrRender(err)) - return - } -} diff --git a/internal/controllers/taskcontroller/create.go b/internal/controllers/taskcontroller/create.go new file mode 100644 index 0000000..7c5f126 --- /dev/null +++ b/internal/controllers/taskcontroller/create.go @@ -0,0 +1,56 @@ +package taskcontroller + +import ( + "encoding/json" + "net/http" + + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/common/errors" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" + "github.com/go-chi/render" +) + +type CreateTaskDTO struct { + Repo string +} + +type CreateTaskResponse struct { + TaskID uint `json:"taskID"` + Status models.TaskStatus `json:"status"` +} + +func (c *CreateTaskResponse) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} + +func (c *TaskController) Create(w http.ResponseWriter, r *http.Request) { + createTaskDto := CreateTaskDTO{} + + if err := json.NewDecoder(r.Body).Decode(&createTaskDto); err != nil { + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusBadRequest, + StatusText: "Invalid JSON", + Err: err, + }) + return + } + + task, err := c.taskService.Create(createTaskDto.Repo) + if err != nil { + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusInternalServerError, + StatusText: "Internal Server Error", + Err: err, + }) + return + } + + response := CreateTaskResponse{ + TaskID: task.ID, + Status: task.Status, + } + + if err := render.Render(w, r, &response); err != nil { + render.Render(w, r, errors.ErrRender(err)) + return + } +} diff --git a/internal/controllers/taskcontroller/upload.go b/internal/controllers/taskcontroller/upload.go new file mode 100644 index 0000000..d60e507 --- /dev/null +++ b/internal/controllers/taskcontroller/upload.go @@ -0,0 +1,75 @@ +package taskcontroller + +import ( + "fmt" + "net/http" + + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/common/errors" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/taskservice" + "github.com/go-chi/chi/v5" + "github.com/go-chi/render" +) + +type TaskUploadResponse struct { + TaskID string `json:"taskID"` + StatusText string `json:"status"` +} + +func (rd *TaskUploadResponse) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} + +func (c *TaskController) Upload(w http.ResponseWriter, r *http.Request) { + taskID := chi.URLParam(r, "taskID") + if taskID == "" { + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusBadRequest, + StatusText: "taskID is required", + }) + return + } + + err := r.ParseMultipartForm(10240 << 20) + if err != nil { + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusBadRequest, + StatusText: "Bad Request", + }) + return + } + + files := r.MultipartForm.File["files"] + for _, fileHeader := range files { + if fileHeader.Size > (1024 << 20) { // Limit each file size to 10MB + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusBadRequest, + StatusText: "File too large", + }) + return + } + } + + err = c.taskService.Upload(&taskservice.TaskUploadInput{ + TaskID: taskID, + Files: files, + }) + if err != nil { + fmt.Println(err) + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusInternalServerError, + StatusText: "Internal Server Error", + Err: err, + }) + return + } + + response := TaskUploadResponse{ + TaskID: taskID, + StatusText: "Success!", + } + + if err := render.Render(w, r, &response); err != nil { + render.Render(w, r, errors.ErrRender(err)) + return + } +} diff --git a/internal/models/db.go b/internal/models/db.go new file mode 100644 index 0000000..0def57a --- /dev/null +++ b/internal/models/db.go @@ -0,0 +1,77 @@ +package models + +import ( + "gorm.io/gorm" +) + +type ALTRepo struct { + gorm.Model + + Name string `gorm:"uniqueIndex"` +} + +type GitRepo struct { + gorm.Model + + Name string `gorm:"uniqueIndex"` +} + +type RPMFile struct { + gorm.Model + + TaskID int + Task Task + + Name string + Arch string +} + +type TaskStatus int + +const ( + StatusPending TaskStatus = iota // 0 + StatusInProgress // 1 + StatusCompleted // 2 + StatusFailed // 3 + StatusCancelled // 4 +) + +type TaskType int + +const ( + // For future purpose + TypeTestOnly TaskType = iota // 0 + TypeUpsert // 1 + // For future purpose + TypeDelete // 2 +) + +type Task struct { + gorm.Model + + Status TaskStatus + Type TaskType + + RepoID uint + Repo *GitRepo + ALTRepoID uint + ALTRepo ALTRepo + + FilesRemoved bool + Files []RPMFile +} + +type GitRepoAltRepoTask struct { + gorm.Model + + RepoID uint `gorm:"uniqueIndex:idx_gr_ar_gitaltrepotask"` + Repo *GitRepo + ALTRepoID uint `gorm:"uniqueIndex:idx_gr_ar_gitaltrepotask"` + ALTRepo *ALTRepo + + LastTaskID *uint + LastTask *Task + + CurrentTaskID *uint + CurrentTask *Task +} diff --git a/internal/models/models.go b/internal/models/models.go deleted file mode 100644 index 5d0d593..0000000 --- a/internal/models/models.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -type FileUpload struct { - TaskID string - - FileName string -} - -type Task struct { - TaskID string `json:"task_id"` - Link string `json:"link"` -} diff --git a/internal/router/router.go b/internal/router/router.go index b27a9a2..f336ed0 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -3,20 +3,22 @@ package router import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/go-co-op/gocron/v2" - "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/controllers/taskcontroller" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/middlewares" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/reposervice" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/taskservice" ) type Router struct { - config *config.Config + app *app.App } -func New(config *config.Config) *Router { +func New(app *app.App) *Router { return &Router{ - config: config, + app: app, } } @@ -25,16 +27,39 @@ func (r *Router) Setup() *chi.Mux { router.Use(middleware.Logger) taskService := taskservice.New( - r.config, + r.app, + ) + + repoService := reposervice.New(r.app) + repoService.ForceUpdate() + + s, _ := gocron.NewScheduler() + defer func() { _ = s.Shutdown() }() + + _, _ = s.NewJob( + gocron.CronJob( + "0 4 * * *", + false, + ), + gocron.NewTask( + func() { + repoService.ForceUpdate() + }, + ), ) taskController := taskcontroller.New( - r.config, + r.app, taskService, ) - router.Route("/task/{taskID}", func(cr chi.Router) { - cr.With(middlewares.CreateAuthGuard(r.config)).Post("/upload", taskController.Upload) + authGuard := middlewares.CreateAuthGuard(r.app.Config) + + router.Route("/tasks", func(taskRouter chi.Router) { + taskRouter.With(authGuard).Post("/", taskController.Create) + taskRouter.Route("/{taskID}", func(sTaskRouter chi.Router) { + sTaskRouter.With(authGuard).Post("/upload", taskController.Upload) + }) }) return router diff --git a/internal/services/reposervice/service.go b/internal/services/reposervice/service.go new file mode 100644 index 0000000..fa48962 --- /dev/null +++ b/internal/services/reposervice/service.go @@ -0,0 +1,156 @@ +package reposervice + +import ( + "fmt" + "os" + "os/exec" + "path" + "strconv" + + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" + "gorm.io/gorm" +) + +type Service struct { + app *app.App +} + +func New(app *app.App) *Service { + return &Service{ + app: app, + } +} + +func createSymlink(target, link string) error { + if _, err := os.Lstat(link); err == nil { + if err := os.Remove(link); err != nil { + return fmt.Errorf("failed to remove existing file or symlink: %w", err) + } + } + + if err := os.Symlink(target, link); err != nil { + return fmt.Errorf("failed to create symlink: %w", err) + } + + return nil +} + +func runGenbasedir(repoDir, arch, repoName string) { + cmd := exec.Command("genbasedir", "--bloat", "--progress", fmt.Sprintf("--topdir=%s", repoDir), arch, repoName) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + fmt.Printf("Failed to run genbasedir for %s: %v\n", arch, err) + os.Exit(1) + } + fmt.Printf("Successfully ran genbasedir for %s\n", arch) +} + +func createRepoDirs(repoDir, repoName, arch string) { + // Create the 'base' directory + baseDir := path.Join(repoDir, arch, "base") + err := os.MkdirAll(baseDir, os.ModePerm) + if err != nil { + fmt.Printf("Failed to create directory %s: %v\n", baseDir, err) + os.Exit(1) + } + + // Create the 'RPMS.' directory + rpmsDir := path.Join(repoDir, arch, fmt.Sprintf("RPMS.%s", repoName)) + err = os.MkdirAll(rpmsDir, os.ModePerm) + if err != nil { + fmt.Printf("Failed to create directory %s: %v\n", rpmsDir, err) + os.Exit(1) + } +} + +func (s *Service) ForceUpdate() { + var tasks []models.Task + altRepo := models.ALTRepo{ + Name: "Sisyphus", + } + + s.app.Db. + Where(&altRepo). + First(&altRepo) + + s.app.Db.Debug(). + Model(&models.GitRepoAltRepoTask{}). + Select("tasks.*"). + Joins("JOIN tasks ON tasks.id = git_repo_alt_repo_tasks.last_task_id"). + Where(&models.GitRepoAltRepoTask{ + ALTRepoID: altRepo.ID, + }). + Preload("Files"). + Find(&tasks) + + repoPath := path.Join(s.app.Config.UploadDir, "future_repo", "Sisyphus") + + os.MkdirAll( + repoPath, + os.ModePerm, + ) + + repoName := "aides" + architectures := []string{"x86_64", "noarch"} + + for _, arch := range architectures { + createRepoDirs(repoPath, repoName, arch) + } + + for _, el := range tasks { + taskPath := path.Join( + s.app.Config.UploadDir, "tasks", strconv.FormatUint(uint64(el.ID), 10), + ) + + for _, fileInfo := range el.Files { + localFilePath := path.Join( + strconv.FormatUint(uint64(el.ID), 10), fileInfo.Name, + ) + symLink := path.Join(s.app.Config.UploadDir, "future_repo", "Sisyphus", fileInfo.Arch, "RPMS.aides", fileInfo.Name) + targetPath := path.Join("../../../../tasks/", localFilePath) + createSymlink(targetPath, symLink) + + fmt.Println(path.Join(taskPath, fileInfo.Name)) + } + } + + for _, arch := range architectures { + runGenbasedir(repoPath, arch, repoName) + } + + s.app.Db.Debug(). + Model(&models.GitRepoAltRepoTask{}). + Where(&models.GitRepoAltRepoTask{ + ALTRepoID: altRepo.ID, + }). + Update( + "current_task_id", gorm.Expr("last_task_id"), + ) + + os.MkdirAll(path.Join(s.app.Config.UploadDir, "repo"), os.ModePerm) + + aPath := path.Join(s.app.Config.UploadDir, "future_repo", "Sisyphus") + bPath := path.Join(s.app.Config.UploadDir, "repo", "Sisyphus") + cPath := path.Join(s.app.Config.UploadDir, "repo", ".Sisyphus") + + if _, err := os.Stat(bPath); err == nil { + fmt.Printf("Moving %s to %s\n", bPath, cPath) + if err := os.Rename(bPath, cPath); err != nil { + } + } else if !os.IsNotExist(err) { + + } + + if err := os.Rename(aPath, bPath); err != nil { + panic(err) + } + + if err := os.RemoveAll(cPath); err != nil { + + } + + os.RemoveAll(path.Join(s.app.Config.UploadDir, "future_repo")) +} diff --git a/internal/services/taskservice/create.go b/internal/services/taskservice/create.go new file mode 100644 index 0000000..aa81c21 --- /dev/null +++ b/internal/services/taskservice/create.go @@ -0,0 +1,31 @@ +package taskservice + +import ( + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" +) + +func (s *Service) Create(repo string) (*models.Task, error) { + taskRepo := models.GitRepo{ + Name: repo, + } + s.app.Db.Debug(). + Where(&taskRepo). + FirstOrCreate(&taskRepo) + + altRepo := models.ALTRepo{ + Name: "Sisyphus", + } + s.app.Db. + Where(&altRepo). + FirstOrCreate(&altRepo) + + task := models.Task{ + RepoID: taskRepo.ID, + ALTRepo: altRepo, + Type: models.TypeUpsert, + } + + result := s.app.Db.Create(&task) + + return &task, result.Error +} diff --git a/internal/services/taskservice/service.go b/internal/services/taskservice/service.go index 6f0f292..7191eef 100644 --- a/internal/services/taskservice/service.go +++ b/internal/services/taskservice/service.go @@ -1,77 +1,99 @@ package taskservice import ( - "io" - "mime/multipart" "os" "path" + "strconv" - "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" ) type Service struct { - config *config.Config + app *app.App } -type TaskUploadInput struct { - TaskID string - Repo string - - Files []*multipart.FileHeader -} - -func New(cfg *config.Config) *Service { +func New(app *app.App) *Service { return &Service{ - config: cfg, + app: app, } } -func (s *Service) Upload(input *TaskUploadInput) error { - repo := input.Repo - taskID := input.TaskID - files := input.Files - - localPath := path.Join(repo, "task", taskID) - taskFolderPath := path.Join(s.config.UploadDir, "extra", localPath) - os.MkdirAll(taskFolderPath, os.ModePerm) - - for _, fileHeader := range files { - file, err := fileHeader.Open() - if err != nil { - return err - } - defer file.Close() - - // Полный путь для файла - filePath := path.Join(taskFolderPath, fileHeader.Filename) - - //Удаляем файл если такой уже существует - if _, err := os.Stat(filePath); err == nil { - err = os.Remove(filePath) - if err != nil { - return err - } - } - // Сохранение файла на сервере - outFile, err := os.Create(filePath) - if err != nil { - return err - } - defer outFile.Close() - - _, err = io.Copy(outFile, file) - if err != nil { - return err - } - // Символическая ссылка - targetPath := path.Join("../extra/", localPath, fileHeader.Filename) - symLink := path.Join(s.config.UploadDir, "out", fileHeader.Filename) - err = createSymlink(targetPath, symLink) - if err != nil { - return err - } - +func (s *Service) onTaskComplete(task *models.Task) error { + if err := s.app.Db.Save(&task).Error; err != nil { + return err } + grart := models.GitRepoAltRepoTask{ + ALTRepoID: 1, + RepoID: task.RepoID, + } + s.app.Db.Debug(). + Where(&grart). + FirstOrCreate(&grart) + + grart.LastTaskID = &task.ID + + if err := s.app.Db.Save(&grart).Error; err != nil { + return err + } + + s.tasksCleanup(&grart, 0) + return nil } + +func (s *Service) tasksCleanup(r *models.GitRepoAltRepoTask, N int) { + excludedTaskIDs := []uint{} + if r.CurrentTaskID != nil { + excludedTaskIDs = append(excludedTaskIDs, *r.CurrentTaskID) + } + if r.LastTaskID != nil { + excludedTaskIDs = append(excludedTaskIDs, *r.LastTaskID) + } + + var lastNTaskIDs []uint + s.app.Db. + Debug(). + Table("tasks"). + Select("id"). + Where("repo_id = ?", r.ID). + Where("status = ?", models.StatusCompleted). + Where("id NOT IN ?", excludedTaskIDs). + Order("created_at DESC"). + Limit(N). + Pluck("id", &lastNTaskIDs) + + excludedTaskIDs = append(excludedTaskIDs, lastNTaskIDs...) + + var taskIDsToDelete []int + s.app.Db. + Debug(). + Model(&models.Task{}). + Select("id"). + Where("repo_id = ?", r.RepoID). + Where("files_removed = ?", false). + Where("id NOT IN ?", excludedTaskIDs). + Pluck("id", &taskIDsToDelete) + + if len(taskIDsToDelete) > 0 { + s.app.Db. + Debug(). + Model(&models.Task{}). + Where("id IN ?", taskIDsToDelete). + Update("files_removed", true) + + for _, id := range taskIDsToDelete { + s.removeTaskFiles(id) + } + } +} + +func (s *Service) removeTaskFiles(taskId int) { + taskFolderPath := path.Join( + s.app.Config.UploadDir, + "tasks", + strconv.Itoa(taskId), + ) + os.RemoveAll(taskFolderPath) +} diff --git a/internal/services/taskservice/upload.go b/internal/services/taskservice/upload.go new file mode 100644 index 0000000..4334721 --- /dev/null +++ b/internal/services/taskservice/upload.go @@ -0,0 +1,126 @@ +package taskservice + +import ( + "errors" + "fmt" + "io" + "mime/multipart" + "os" + "os/exec" + "path" + "strconv" + "strings" + + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" + "gorm.io/gorm" +) + +type TaskUploadInput struct { + TaskID string + + Files []*multipart.FileHeader +} + +func getRPMArchitecture(filePath string) (string, error) { + cmd := exec.Command("rpm", "-qp", "--queryformat", "%{ARCH}", filePath) + + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("ошибка при выполнении команды rpm: %v", err) + } + + arch := strings.TrimSpace(string(output)) + return arch, nil +} + +func (s *Service) Upload(input *TaskUploadInput) error { + taskID, err := strconv.Atoi(input.TaskID) + if err != nil { + return err + } + files := input.Files + + task := models.Task{} + result := s.app.Db.Preload("Repo").Where( + "id = ?", taskID, + ).First(&task) + /*.Where( + "status = ?", models.StatusPending, + )*/ + + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return result.Error + } + if result.Error != nil { + return result.Error + } + + localPath := path.Join(input.TaskID) + taskFolderPath := path.Join(s.app.Config.UploadDir, "tasks", localPath) + os.MkdirAll(taskFolderPath, os.ModePerm) + + for _, fileHeader := range files { + file, err := fileHeader.Open() + if err != nil { + return err + } + defer file.Close() + + if !strings.HasSuffix(fileHeader.Filename, ".rpm") { + return fmt.Errorf("invalid file type: only .rpm files are allowed") + } + + // Полный путь для файла + filePath := path.Join(taskFolderPath, fileHeader.Filename) + + //Удаляем файл если такой уже существует + if _, err := os.Stat(filePath); err == nil { + err = os.Remove(filePath) + if err != nil { + return err + } + } + // Сохранение файла на сервере + outFile, err := os.Create(filePath) + if err != nil { + return err + } + defer outFile.Close() + + _, err = io.Copy(outFile, file) + if err != nil { + return err + } + + arch, err := getRPMArchitecture(filePath) + if err != nil { + return err + } + + f := models.RPMFile{ + Name: fileHeader.Filename, + Arch: arch, + Task: task, + } + + if err := s.app.Db.Save(&f).Error; err != nil { + return err + } + + // Символическая ссылка + /* + targetPath := path.Join("../../../../tasks/", localPath, fileHeader.Filename) + symLink := path.Join(s.app.Config.UploadDir, "repo/Sisyphus", arch, "RPMS.aides", fileHeader.Filename) + err = createSymlink(targetPath, symLink) + if err != nil { + return err + } + */ + + } + + task.Status = models.StatusCompleted + s.onTaskComplete(&task) + + return nil +}