From cbf156f01e010264aa8506651af9bdc2e4327b4a Mon Sep 17 00:00:00 2001 From: Maxim Slipenko Date: Thu, 12 Dec 2024 23:36:38 +0300 Subject: [PATCH 1/7] feat: add database --- cmd/aides-repo-api/main.go | 13 +-- go.mod | 10 ++- go.sum | 12 +++ internal/app/app.go | 31 +++++++ .../controllers/taskcontroller/controller.go | 88 +------------------ internal/controllers/taskcontroller/create.go | 56 ++++++++++++ internal/controllers/taskcontroller/upload.go | 87 ++++++++++++++++++ internal/models/db.go | 57 ++++++++++++ internal/models/models.go | 12 --- internal/router/router.go | 21 +++-- internal/services/taskservice/create.go | 24 +++++ internal/services/taskservice/service.go | 38 ++++++-- 12 files changed, 330 insertions(+), 119 deletions(-) create mode 100644 internal/app/app.go create mode 100644 internal/controllers/taskcontroller/create.go create mode 100644 internal/controllers/taskcontroller/upload.go create mode 100644 internal/models/db.go delete mode 100644 internal/models/models.go create mode 100644 internal/services/taskservice/create.go diff --git a/cmd/aides-repo-api/main.go b/cmd/aides-repo-api/main.go index 8ff30a0..bd476bb 100644 --- a/cmd/aides-repo-api/main.go +++ b/cmd/aides-repo-api/main.go @@ -4,16 +4,19 @@ import ( "log" "net/http" - "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/router" ) func main() { - config := config.New() + app, err := app.New() + if err != nil { + panic(err) + } // Конфигурация сервера - router := router.New(config).Setup() + router := router.New(app).Setup() - log.Printf("Сервер запущен на порту: %s", config.Port) - http.ListenAndServe(":"+config.Port, router) + log.Printf("Сервер запущен на порту: %s", app.Config.Port) + http.ListenAndServe(":"+app.Config.Port, router) } diff --git a/go.mod b/go.mod index fc13160..fcc8a8c 100644 --- a/go.mod +++ b/go.mod @@ -8,4 +8,12 @@ require ( github.com/go-chi/render v1.0.3 ) -require github.com/ajg/form v1.5.1 // indirect +require ( + github.com/ajg/form v1.5.1 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect + github.com/mattn/go-sqlite3 v1.14.24 // indirect + golang.org/x/text v0.21.0 // indirect + gorm.io/driver/sqlite v1.5.7 // indirect + gorm.io/gorm v1.25.12 // indirect +) diff --git a/go.sum b/go.sum index 55110ff..5acd7b7 100644 --- a/go.sum +++ b/go.sum @@ -6,3 +6,15 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= +github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I= +gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= +gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= +gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 0000000..4bb8cac --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,31 @@ +package app + +import ( + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +type App struct { + Db *gorm.DB + Config *config.Config +} + +func New() (*App, error) { + db, err := gorm.Open(sqlite.Open("app.db"), &gorm.Config{}) + if err != nil { + return nil, err + } + + db.AutoMigrate(&models.Task{}, &models.GitRepoAltRepoTask{}) + + db.FirstOrCreate(&models.ALTRepo{ + Name: "Sisyphus", + }) + + return &App{ + Config: config.New(), + Db: db, + }, nil +} diff --git a/internal/controllers/taskcontroller/controller.go b/internal/controllers/taskcontroller/controller.go index 24702e4..46efa1b 100644 --- a/internal/controllers/taskcontroller/controller.go +++ b/internal/controllers/taskcontroller/controller.go @@ -1,98 +1,18 @@ package taskcontroller import ( - "net/http" - - "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/common/errors" - "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/taskservice" - "github.com/go-chi/chi/v5" - "github.com/go-chi/render" ) -type TaskUploadResponse struct { - TaskID string `json:"taskID"` - Repo string `json:"repo"` - StatusText string `json:"status"` -} - -func (rd *TaskUploadResponse) Render(w http.ResponseWriter, r *http.Request) error { - render.Status(r, http.StatusOK) - return nil -} - type TaskController struct { - config *config.Config + app *app.App taskService *taskservice.Service } -func New(cfg *config.Config, taskService *taskservice.Service) *TaskController { +func New(app *app.App, taskService *taskservice.Service) *TaskController { return &TaskController{ - config: cfg, + app: app, taskService: taskService, } } - -func (c *TaskController) Upload(w http.ResponseWriter, r *http.Request) { - taskID := chi.URLParam(r, "taskID") - if taskID == "" { - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusBadRequest, - StatusText: "taskID is required", - }) - return - } - - err := r.ParseMultipartForm(10240 << 20) - if err != nil { - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusBadRequest, - StatusText: "Bad Request", - }) - return - } - - repo := r.FormValue("repo") - if repo == "" { - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusBadRequest, - StatusText: "Missing required repo field", - }) - return - } - - files := r.MultipartForm.File["files"] - for _, fileHeader := range files { - if fileHeader.Size > (1024 << 20) { // Limit each file size to 10MB - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusBadRequest, - StatusText: "File too large", - }) - return - } - } - - err = c.taskService.Upload(&taskservice.TaskUploadInput{ - TaskID: taskID, - Repo: repo, - Files: files, - }) - if err != nil { - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusInternalServerError, - StatusText: "Internal Server Error", - Err: err, - }) - } - - response := TaskUploadResponse{ - TaskID: taskID, - Repo: repo, - StatusText: "Success!", - } - - if err := render.Render(w, r, &response); err != nil { - render.Render(w, r, errors.ErrRender(err)) - return - } -} diff --git a/internal/controllers/taskcontroller/create.go b/internal/controllers/taskcontroller/create.go new file mode 100644 index 0000000..8184a86 --- /dev/null +++ b/internal/controllers/taskcontroller/create.go @@ -0,0 +1,56 @@ +package taskcontroller + +import ( + "encoding/json" + "net/http" + + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/common/errors" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" + "github.com/go-chi/render" +) + +type CreateTaskDTO struct { + Repo string +} + +type CreateTaskResponse struct { + TaskID int `json:"taskID"` + Status models.TaskStatus `json:"status"` +} + +func (c *CreateTaskResponse) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} + +func (c *TaskController) Create(w http.ResponseWriter, r *http.Request) { + createTaskDto := CreateTaskDTO{} + + if err := json.NewDecoder(r.Body).Decode(&createTaskDto); err != nil { + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusBadRequest, + StatusText: "Invalid JSON", + Err: err, + }) + return + } + + task, err := c.taskService.Create(createTaskDto.Repo) + if err != nil { + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusInternalServerError, + StatusText: "Internal Server Error", + Err: err, + }) + return + } + + response := CreateTaskResponse{ + TaskID: task.ID, + Status: task.Status, + } + + if err := render.Render(w, r, &response); err != nil { + render.Render(w, r, errors.ErrRender(err)) + return + } +} diff --git a/internal/controllers/taskcontroller/upload.go b/internal/controllers/taskcontroller/upload.go new file mode 100644 index 0000000..9700716 --- /dev/null +++ b/internal/controllers/taskcontroller/upload.go @@ -0,0 +1,87 @@ +package taskcontroller + +import ( + "fmt" + "net/http" + + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/common/errors" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/taskservice" + "github.com/go-chi/chi/v5" + "github.com/go-chi/render" +) + +type TaskUploadResponse struct { + TaskID string `json:"taskID"` + Repo string `json:"repo"` + StatusText string `json:"status"` +} + +func (rd *TaskUploadResponse) Render(w http.ResponseWriter, r *http.Request) error { + return nil +} + +func (c *TaskController) Upload(w http.ResponseWriter, r *http.Request) { + taskID := chi.URLParam(r, "taskID") + if taskID == "" { + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusBadRequest, + StatusText: "taskID is required", + }) + return + } + + err := r.ParseMultipartForm(10240 << 20) + if err != nil { + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusBadRequest, + StatusText: "Bad Request", + }) + return + } + + repo := r.FormValue("repo") + if repo == "" { + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusBadRequest, + StatusText: "Missing required repo field", + }) + return + } + + files := r.MultipartForm.File["files"] + for _, fileHeader := range files { + if fileHeader.Size > (1024 << 20) { // Limit each file size to 10MB + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusBadRequest, + StatusText: "File too large", + }) + return + } + } + + err = c.taskService.Upload(&taskservice.TaskUploadInput{ + TaskID: taskID, + Repo: repo, + Files: files, + }) + if err != nil { + fmt.Println(err) + render.Render(w, r, &errors.ErrResponse{ + HTTPStatusCode: http.StatusInternalServerError, + StatusText: "Internal Server Error", + Err: err, + }) + return + } + + response := TaskUploadResponse{ + TaskID: taskID, + Repo: repo, + StatusText: "Success!", + } + + if err := render.Render(w, r, &response); err != nil { + render.Render(w, r, errors.ErrRender(err)) + return + } +} diff --git a/internal/models/db.go b/internal/models/db.go new file mode 100644 index 0000000..0a95f9b --- /dev/null +++ b/internal/models/db.go @@ -0,0 +1,57 @@ +package models + +import "gorm.io/gorm" + +type ALTRepo struct { + gorm.Model + + ID int + Name string `gorm:"uniqueIndex"` +} + +type GitRepo struct { + ID int + Name string `gorm:"uniqueIndex"` +} + +type RPMFiles struct { + TaskID int + Name string +} + +type TaskStatus int + +const ( + StatusPending TaskStatus = iota // 0 + StatusInProgress // 1 + StatusCompleted // 2 + StatusFailed // 3 + StatusCancelled // 4 +) + +type Task struct { + gorm.Model + + ID int + Status TaskStatus + + RepoID int + Repo GitRepo + ALTRepoID int + ALTRepo ALTRepo + + RPMFiles []RPMFiles +} + +type GitRepoAltRepoTask struct { + gorm.Model + + ID int + + RepoID int `gorm:"uniqueIndex:idx_gr_ar_gitaltrepotask"` + Repo GitRepo + ALTRepoID int `gorm:"uniqueIndex:idx_gr_ar_gitaltrepotask"` + ALTRepo ALTRepo + TaskID int + Task Task +} diff --git a/internal/models/models.go b/internal/models/models.go deleted file mode 100644 index 5d0d593..0000000 --- a/internal/models/models.go +++ /dev/null @@ -1,12 +0,0 @@ -package models - -type FileUpload struct { - TaskID string - - FileName string -} - -type Task struct { - TaskID string `json:"task_id"` - Link string `json:"link"` -} diff --git a/internal/router/router.go b/internal/router/router.go index b27a9a2..85d1bfb 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -4,19 +4,19 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" - "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/controllers/taskcontroller" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/middlewares" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/taskservice" ) type Router struct { - config *config.Config + app *app.App } -func New(config *config.Config) *Router { +func New(app *app.App) *Router { return &Router{ - config: config, + app: app, } } @@ -25,16 +25,21 @@ func (r *Router) Setup() *chi.Mux { router.Use(middleware.Logger) taskService := taskservice.New( - r.config, + r.app, ) taskController := taskcontroller.New( - r.config, + r.app, taskService, ) - router.Route("/task/{taskID}", func(cr chi.Router) { - cr.With(middlewares.CreateAuthGuard(r.config)).Post("/upload", taskController.Upload) + authGuard := middlewares.CreateAuthGuard(r.app.Config) + + router.Route("/tasks", func(taskRouter chi.Router) { + taskRouter.With(authGuard).Post("/", taskController.Create) + taskRouter.Route("/{taskID}", func(sTaskRouter chi.Router) { + sTaskRouter.With(authGuard).Post("/upload", taskController.Upload) + }) }) return router diff --git a/internal/services/taskservice/create.go b/internal/services/taskservice/create.go new file mode 100644 index 0000000..8b32f12 --- /dev/null +++ b/internal/services/taskservice/create.go @@ -0,0 +1,24 @@ +package taskservice + +import "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" + +func (s *Service) Create(repo string) (*models.Task, error) { + taskRepo := models.GitRepo{ + Name: repo, + } + s.app.Db.FirstOrCreate(&taskRepo) + + altRepo := models.ALTRepo{ + Name: "Sisyphus", + } + s.app.Db.FirstOrCreate(&altRepo) + + task := models.Task{ + RepoID: taskRepo.ID, + ALTRepo: altRepo, + } + + result := s.app.Db.Create(&task) + + return &task, result.Error +} diff --git a/internal/services/taskservice/service.go b/internal/services/taskservice/service.go index 6f0f292..68a0065 100644 --- a/internal/services/taskservice/service.go +++ b/internal/services/taskservice/service.go @@ -1,16 +1,19 @@ package taskservice import ( + "fmt" "io" "mime/multipart" "os" + "os/exec" "path" + "strings" - "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" ) type Service struct { - config *config.Config + app *app.App } type TaskUploadInput struct { @@ -20,19 +23,30 @@ type TaskUploadInput struct { Files []*multipart.FileHeader } -func New(cfg *config.Config) *Service { +func New(app *app.App) *Service { return &Service{ - config: cfg, + app: app, } } +func getRPMArchitecture(filePath string) (string, error) { + cmd := exec.Command("rpm", "-qp", "--queryformat", "%{ARCH}", filePath) + + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("ошибка при выполнении команды rpm: %v", err) + } + + arch := strings.TrimSpace(string(output)) + return arch, nil +} + func (s *Service) Upload(input *TaskUploadInput) error { - repo := input.Repo taskID := input.TaskID files := input.Files - localPath := path.Join(repo, "task", taskID) - taskFolderPath := path.Join(s.config.UploadDir, "extra", localPath) + localPath := path.Join(taskID) + taskFolderPath := path.Join(s.app.Config.UploadDir, "tasks", localPath) os.MkdirAll(taskFolderPath, os.ModePerm) for _, fileHeader := range files { @@ -63,9 +77,15 @@ func (s *Service) Upload(input *TaskUploadInput) error { if err != nil { return err } + + arch, err := getRPMArchitecture(filePath) + if err != nil { + return err + } + // Символическая ссылка - targetPath := path.Join("../extra/", localPath, fileHeader.Filename) - symLink := path.Join(s.config.UploadDir, "out", fileHeader.Filename) + targetPath := path.Join("../../../../tasks/", localPath, fileHeader.Filename) + symLink := path.Join(s.app.Config.UploadDir, "repo/Sisyphus", arch, "RPMS.aides", fileHeader.Filename) err = createSymlink(targetPath, symLink) if err != nil { return err -- 2.45.2 From 32b99f4f112dc36c3247342fe9484f178a912d68 Mon Sep 17 00:00:00 2001 From: Maxim Slipenko Date: Thu, 12 Dec 2024 23:51:13 +0300 Subject: [PATCH 2/7] feat: add task type --- .gitignore | 4 +++- internal/models/db.go | 11 +++++++++++ internal/services/taskservice/create.go | 1 + 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index c7348ad..a450265 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,6 @@ fabric.properties .idea/**/azureSettings.xml # Игнорируем папку uploads -uploads \ No newline at end of file +uploads + +app.db \ No newline at end of file diff --git a/internal/models/db.go b/internal/models/db.go index 0a95f9b..6529ca6 100644 --- a/internal/models/db.go +++ b/internal/models/db.go @@ -29,11 +29,22 @@ const ( StatusCancelled // 4 ) +type TaskType int + +const ( + // For future purpose + TypeTestOnly TaskType = iota // 0 + TypeUpsert // 1 + // For future purpose + TypeDelete // 2 +) + type Task struct { gorm.Model ID int Status TaskStatus + Type TaskType RepoID int Repo GitRepo diff --git a/internal/services/taskservice/create.go b/internal/services/taskservice/create.go index 8b32f12..d178359 100644 --- a/internal/services/taskservice/create.go +++ b/internal/services/taskservice/create.go @@ -16,6 +16,7 @@ func (s *Service) Create(repo string) (*models.Task, error) { task := models.Task{ RepoID: taskRepo.ID, ALTRepo: altRepo, + Type: models.TypeUpsert, } result := s.app.Db.Create(&task) -- 2.45.2 From 0fedac8e9374ae6b2d8a38e83587d5c86366c7b4 Mon Sep 17 00:00:00 2001 From: Maxim Slipenko Date: Fri, 13 Dec 2024 10:10:12 +0300 Subject: [PATCH 3/7] fix: allow upload only pending tasks --- internal/models/db.go | 9 +- internal/services/taskservice/service.go | 73 --------------- internal/services/taskservice/upload.go | 114 +++++++++++++++++++++++ 3 files changed, 120 insertions(+), 76 deletions(-) create mode 100644 internal/services/taskservice/upload.go diff --git a/internal/models/db.go b/internal/models/db.go index 6529ca6..9e05c8f 100644 --- a/internal/models/db.go +++ b/internal/models/db.go @@ -1,6 +1,8 @@ package models -import "gorm.io/gorm" +import ( + "gorm.io/gorm" +) type ALTRepo struct { gorm.Model @@ -14,9 +16,10 @@ type GitRepo struct { Name string `gorm:"uniqueIndex"` } -type RPMFiles struct { +type RPMFile struct { TaskID int Name string + Arch string } type TaskStatus int @@ -51,7 +54,7 @@ type Task struct { ALTRepoID int ALTRepo ALTRepo - RPMFiles []RPMFiles + Files []RPMFile } type GitRepoAltRepoTask struct { diff --git a/internal/services/taskservice/service.go b/internal/services/taskservice/service.go index 68a0065..6b0eccc 100644 --- a/internal/services/taskservice/service.go +++ b/internal/services/taskservice/service.go @@ -1,13 +1,7 @@ package taskservice import ( - "fmt" - "io" "mime/multipart" - "os" - "os/exec" - "path" - "strings" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" ) @@ -28,70 +22,3 @@ func New(app *app.App) *Service { app: app, } } - -func getRPMArchitecture(filePath string) (string, error) { - cmd := exec.Command("rpm", "-qp", "--queryformat", "%{ARCH}", filePath) - - output, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("ошибка при выполнении команды rpm: %v", err) - } - - arch := strings.TrimSpace(string(output)) - return arch, nil -} - -func (s *Service) Upload(input *TaskUploadInput) error { - taskID := input.TaskID - files := input.Files - - localPath := path.Join(taskID) - taskFolderPath := path.Join(s.app.Config.UploadDir, "tasks", localPath) - os.MkdirAll(taskFolderPath, os.ModePerm) - - for _, fileHeader := range files { - file, err := fileHeader.Open() - if err != nil { - return err - } - defer file.Close() - - // Полный путь для файла - filePath := path.Join(taskFolderPath, fileHeader.Filename) - - //Удаляем файл если такой уже существует - if _, err := os.Stat(filePath); err == nil { - err = os.Remove(filePath) - if err != nil { - return err - } - } - // Сохранение файла на сервере - outFile, err := os.Create(filePath) - if err != nil { - return err - } - defer outFile.Close() - - _, err = io.Copy(outFile, file) - if err != nil { - return err - } - - arch, err := getRPMArchitecture(filePath) - if err != nil { - return err - } - - // Символическая ссылка - targetPath := path.Join("../../../../tasks/", localPath, fileHeader.Filename) - symLink := path.Join(s.app.Config.UploadDir, "repo/Sisyphus", arch, "RPMS.aides", fileHeader.Filename) - err = createSymlink(targetPath, symLink) - if err != nil { - return err - } - - } - - return nil -} diff --git a/internal/services/taskservice/upload.go b/internal/services/taskservice/upload.go new file mode 100644 index 0000000..4bd8973 --- /dev/null +++ b/internal/services/taskservice/upload.go @@ -0,0 +1,114 @@ +package taskservice + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "path" + "strconv" + "strings" + + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" + "gorm.io/gorm" +) + +func getRPMArchitecture(filePath string) (string, error) { + cmd := exec.Command("rpm", "-qp", "--queryformat", "%{ARCH}", filePath) + + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("ошибка при выполнении команды rpm: %v", err) + } + + arch := strings.TrimSpace(string(output)) + return arch, nil +} + +func (s *Service) Upload(input *TaskUploadInput) error { + taskID, err := strconv.Atoi(input.TaskID) + if err != nil { + return err + } + files := input.Files + + task := models.Task{} + result := s.app.Db.Where( + "id = ?", taskID, + ).Where( + "status = ?", models.StatusPending, + ).First(&task) + + if errors.Is(result.Error, gorm.ErrRecordNotFound) { + return result.Error + } + if result.Error != nil { + return result.Error + } + + fmt.Printf("%v", task.Status) + + localPath := path.Join(input.TaskID) + taskFolderPath := path.Join(s.app.Config.UploadDir, "tasks", localPath) + os.MkdirAll(taskFolderPath, os.ModePerm) + + for _, fileHeader := range files { + file, err := fileHeader.Open() + if err != nil { + return err + } + defer file.Close() + + if !strings.HasSuffix(fileHeader.Filename, ".rpm") { + return fmt.Errorf("invalid file type: only .rpm files are allowed") + } + + // Полный путь для файла + filePath := path.Join(taskFolderPath, fileHeader.Filename) + + //Удаляем файл если такой уже существует + if _, err := os.Stat(filePath); err == nil { + err = os.Remove(filePath) + if err != nil { + return err + } + } + // Сохранение файла на сервере + outFile, err := os.Create(filePath) + if err != nil { + return err + } + defer outFile.Close() + + _, err = io.Copy(outFile, file) + if err != nil { + return err + } + + arch, err := getRPMArchitecture(filePath) + if err != nil { + return err + } + + fmt.Println(arch) + + // Символическая ссылка + /* + targetPath := path.Join("../../../../tasks/", localPath, fileHeader.Filename) + symLink := path.Join(s.app.Config.UploadDir, "repo/Sisyphus", arch, "RPMS.aides", fileHeader.Filename) + err = createSymlink(targetPath, symLink) + if err != nil { + return err + } + */ + + } + + task.Status = models.StatusCompleted + if err := s.app.Db.Save(&task).Error; err != nil { + return err + } + + return nil +} -- 2.45.2 From 26c751ccb730e380813de90a71d7eb745fedf324 Mon Sep 17 00:00:00 2001 From: Maxim Slipenko Date: Fri, 13 Dec 2024 13:52:15 +0300 Subject: [PATCH 4/7] feat: add repo creation --- internal/app/app.go | 2 +- internal/controllers/taskcontroller/create.go | 2 +- internal/models/db.go | 38 ++--- internal/router/router.go | 4 + internal/services/reposervice/service.go | 134 ++++++++++++++++++ internal/services/taskservice/service.go | 91 ++++++++++-- internal/services/taskservice/upload.go | 31 ++-- 7 files changed, 267 insertions(+), 35 deletions(-) create mode 100644 internal/services/reposervice/service.go diff --git a/internal/app/app.go b/internal/app/app.go index 4bb8cac..6f9878a 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -18,7 +18,7 @@ func New() (*App, error) { return nil, err } - db.AutoMigrate(&models.Task{}, &models.GitRepoAltRepoTask{}) + db.AutoMigrate(&models.Task{}, &models.GitRepoAltRepoTask{}, &models.RPMFile{}) db.FirstOrCreate(&models.ALTRepo{ Name: "Sisyphus", diff --git a/internal/controllers/taskcontroller/create.go b/internal/controllers/taskcontroller/create.go index 8184a86..7c5f126 100644 --- a/internal/controllers/taskcontroller/create.go +++ b/internal/controllers/taskcontroller/create.go @@ -14,7 +14,7 @@ type CreateTaskDTO struct { } type CreateTaskResponse struct { - TaskID int `json:"taskID"` + TaskID uint `json:"taskID"` Status models.TaskStatus `json:"status"` } diff --git a/internal/models/db.go b/internal/models/db.go index 9e05c8f..46a995a 100644 --- a/internal/models/db.go +++ b/internal/models/db.go @@ -7,19 +7,23 @@ import ( type ALTRepo struct { gorm.Model - ID int Name string `gorm:"uniqueIndex"` } type GitRepo struct { - ID int + gorm.Model + Name string `gorm:"uniqueIndex"` } type RPMFile struct { + gorm.Model + TaskID int - Name string - Arch string + Task Task + + Name string + Arch string } type TaskStatus int @@ -45,27 +49,29 @@ const ( type Task struct { gorm.Model - ID int Status TaskStatus Type TaskType - RepoID int - Repo GitRepo - ALTRepoID int + RepoID uint + Repo *GitRepo + ALTRepoID uint ALTRepo ALTRepo - Files []RPMFile + FilesRemoved bool + Files []RPMFile } type GitRepoAltRepoTask struct { gorm.Model - ID int + RepoID uint `gorm:"uniqueIndex:idx_gr_ar_gitaltrepotask"` + Repo *GitRepo + ALTRepoID uint `gorm:"uniqueIndex:idx_gr_ar_gitaltrepotask"` + ALTRepo *ALTRepo - RepoID int `gorm:"uniqueIndex:idx_gr_ar_gitaltrepotask"` - Repo GitRepo - ALTRepoID int `gorm:"uniqueIndex:idx_gr_ar_gitaltrepotask"` - ALTRepo ALTRepo - TaskID int - Task Task + LastTaskID uint + LastTask *Task + + CurrentTaskID uint + CurrentTask Task } diff --git a/internal/router/router.go b/internal/router/router.go index 85d1bfb..21cdfb6 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -7,6 +7,7 @@ import ( "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/controllers/taskcontroller" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/middlewares" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/reposervice" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/taskservice" ) @@ -28,6 +29,9 @@ func (r *Router) Setup() *chi.Mux { r.app, ) + repoService := reposervice.New(r.app) + repoService.ForceUpdate() + taskController := taskcontroller.New( r.app, taskService, diff --git a/internal/services/reposervice/service.go b/internal/services/reposervice/service.go new file mode 100644 index 0000000..bac9519 --- /dev/null +++ b/internal/services/reposervice/service.go @@ -0,0 +1,134 @@ +package reposervice + +import ( + "fmt" + "os" + "os/exec" + "path" + "strconv" + + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" + "gorm.io/gorm" +) + +type Service struct { + app *app.App +} + +func New(app *app.App) *Service { + return &Service{ + app: app, + } +} + +func createSymlink(target, link string) error { + if _, err := os.Lstat(link); err == nil { + if err := os.Remove(link); err != nil { + return fmt.Errorf("failed to remove existing file or symlink: %w", err) + } + } + + if err := os.Symlink(target, link); err != nil { + return fmt.Errorf("failed to create symlink: %w", err) + } + + return nil +} + +func runGenbasedir(repoDir, arch, repoName string) { + cmd := exec.Command("genbasedir", "--bloat", "--progress", fmt.Sprintf("--topdir=%s", repoDir), arch, repoName) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + fmt.Printf("Failed to run genbasedir for %s: %v\n", arch, err) + os.Exit(1) + } + fmt.Printf("Successfully ran genbasedir for %s\n", arch) +} + +func createRepoDirs(repoDir, repoName, arch string) { + // Create the 'base' directory + baseDir := path.Join(repoDir, arch, "base") + err := os.MkdirAll(baseDir, os.ModePerm) + if err != nil { + fmt.Printf("Failed to create directory %s: %v\n", baseDir, err) + os.Exit(1) + } + + // Create the 'RPMS.' directory + rpmsDir := path.Join(repoDir, arch, fmt.Sprintf("RPMS.%s", repoName)) + err = os.MkdirAll(rpmsDir, os.ModePerm) + if err != nil { + fmt.Printf("Failed to create directory %s: %v\n", rpmsDir, err) + os.Exit(1) + } +} + +func (s *Service) ForceUpdate() { + var tasks []models.Task + altRepo := models.ALTRepo{ + Name: "Sisyphus", + } + + s.app.Db. + Where(&altRepo). + First(&altRepo) + + s.app.Db.Debug(). + Model(&models.GitRepoAltRepoTask{}). + Select("tasks.*"). + Joins("JOIN tasks ON tasks.id = git_repo_alt_repo_tasks.last_task_id"). + Where(&models.GitRepoAltRepoTask{ + ALTRepoID: altRepo.ID, + }). + Preload("Files"). + Find(&tasks) + + repoPath := path.Join(s.app.Config.UploadDir, "future_repo", "Sisyphus") + + os.MkdirAll( + repoPath, + os.ModePerm, + ) + + repoName := "aides" + architectures := []string{"x86_64", "noarch"} + + for _, arch := range architectures { + createRepoDirs(repoPath, repoName, arch) + } + + for _, el := range tasks { + fmt.Println(el.Files) + + taskPath := path.Join( + s.app.Config.UploadDir, "tasks", strconv.FormatUint(uint64(el.ID), 10), + ) + + for _, fileInfo := range el.Files { + localFilePath := path.Join( + strconv.FormatUint(uint64(el.ID), 10), fileInfo.Name, + ) + symLink := path.Join(s.app.Config.UploadDir, "future_repo", "Sisyphus", fileInfo.Arch, "RPMS.aides", fileInfo.Name) + targetPath := path.Join("../../../../tasks/", localFilePath) + createSymlink(targetPath, symLink) + + fmt.Println(path.Join(taskPath, fileInfo.Name)) + } + } + + for _, arch := range architectures { + runGenbasedir(repoPath, arch, repoName) + } + + s.app.Db.Debug(). + Model(&models.GitRepoAltRepoTask{}). + Where(&models.GitRepoAltRepoTask{ + ALTRepoID: altRepo.ID, + }). + Update( + "current_task_id", gorm.Expr("last_task_id"), + ) +} diff --git a/internal/services/taskservice/service.go b/internal/services/taskservice/service.go index 6b0eccc..c0e78f3 100644 --- a/internal/services/taskservice/service.go +++ b/internal/services/taskservice/service.go @@ -1,24 +1,99 @@ package taskservice import ( - "mime/multipart" + "os" + "path" + "strconv" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" ) type Service struct { app *app.App } -type TaskUploadInput struct { - TaskID string - Repo string - - Files []*multipart.FileHeader -} - func New(app *app.App) *Service { return &Service{ app: app, } } + +func (s *Service) onTaskComplete(task *models.Task) error { + if err := s.app.Db.Save(&task).Error; err != nil { + return err + } + + grart := models.GitRepoAltRepoTask{ + ALTRepoID: 1, + Repo: task.Repo, + } + s.app.Db.Debug().Where( + &grart, + ).FirstOrCreate(&grart) + + grart.LastTaskID = task.ID + + if err := s.app.Db.Save(&grart).Error; err != nil { + return err + } + + s.tasksCleanup(&grart, 0) + + return nil +} + +func (s *Service) tasksCleanup(r *models.GitRepoAltRepoTask, N int) { + excludedTaskIDs := []uint{} + if r.CurrentTaskID != 0 { + excludedTaskIDs = append(excludedTaskIDs, r.CurrentTaskID) + } + if r.LastTaskID != 0 { + excludedTaskIDs = append(excludedTaskIDs, r.LastTaskID) + } + + var lastNTaskIDs []uint + s.app.Db. + Debug(). + Table("tasks"). + Select("id"). + Where("repo_id = ?", r.ID). + Where("status = ?", models.StatusCompleted). + Where("id NOT IN ?", excludedTaskIDs). + Order("created_at DESC"). + Limit(N). + Pluck("id", &lastNTaskIDs) + + excludedTaskIDs = append(excludedTaskIDs, lastNTaskIDs...) + + var taskIDsToDelete []int + s.app.Db. + Debug(). + Model(&models.Task{}). + Select("id"). + Where("repo_id = ?", r.RepoID). + Where("files_removed = ?", false). + Where("id NOT IN ?", excludedTaskIDs). + Pluck("id", &taskIDsToDelete) + + if len(taskIDsToDelete) > 0 { + s.app.Db. + Debug(). + Model(&models.Task{}). + Where("id IN ?", taskIDsToDelete). + Update("files_removed", true) + + for _, id := range taskIDsToDelete { + s.removeTaskFiles(id) + } + } +} + +func (s *Service) removeTaskFiles(taskId int) { + taskFolderPath := path.Join( + s.app.Config.UploadDir, + "tasks", + strconv.Itoa(taskId), + ) + os.RemoveAll(taskFolderPath) +} diff --git a/internal/services/taskservice/upload.go b/internal/services/taskservice/upload.go index 4bd8973..0465c32 100644 --- a/internal/services/taskservice/upload.go +++ b/internal/services/taskservice/upload.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io" + "mime/multipart" "os" "os/exec" "path" @@ -14,6 +15,13 @@ import ( "gorm.io/gorm" ) +type TaskUploadInput struct { + TaskID string + Repo string + + Files []*multipart.FileHeader +} + func getRPMArchitecture(filePath string) (string, error) { cmd := exec.Command("rpm", "-qp", "--queryformat", "%{ARCH}", filePath) @@ -34,11 +42,12 @@ func (s *Service) Upload(input *TaskUploadInput) error { files := input.Files task := models.Task{} - result := s.app.Db.Where( + result := s.app.Db.Preload("Repo").Where( "id = ?", taskID, - ).Where( - "status = ?", models.StatusPending, ).First(&task) + /*.Where( + "status = ?", models.StatusPending, + )*/ if errors.Is(result.Error, gorm.ErrRecordNotFound) { return result.Error @@ -47,8 +56,6 @@ func (s *Service) Upload(input *TaskUploadInput) error { return result.Error } - fmt.Printf("%v", task.Status) - localPath := path.Join(input.TaskID) taskFolderPath := path.Join(s.app.Config.UploadDir, "tasks", localPath) os.MkdirAll(taskFolderPath, os.ModePerm) @@ -91,7 +98,15 @@ func (s *Service) Upload(input *TaskUploadInput) error { return err } - fmt.Println(arch) + f := models.RPMFile{ + Name: fileHeader.Filename, + Arch: arch, + Task: task, + } + + if err := s.app.Db.Save(&f).Error; err != nil { + return err + } // Символическая ссылка /* @@ -106,9 +121,7 @@ func (s *Service) Upload(input *TaskUploadInput) error { } task.Status = models.StatusCompleted - if err := s.app.Db.Save(&task).Error; err != nil { - return err - } + s.onTaskComplete(&task) return nil } -- 2.45.2 From f2ef06ca819ff37d6b2b16d65a1bff5b5e54ce4d Mon Sep 17 00:00:00 2001 From: Maxim Slipenko Date: Fri, 13 Dec 2024 14:04:27 +0300 Subject: [PATCH 5/7] feat: add cron --- internal/router/router.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/router/router.go b/internal/router/router.go index 21cdfb6..f336ed0 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -3,6 +3,7 @@ package router import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/go-co-op/gocron/v2" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/app" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/controllers/taskcontroller" @@ -32,6 +33,21 @@ func (r *Router) Setup() *chi.Mux { repoService := reposervice.New(r.app) repoService.ForceUpdate() + s, _ := gocron.NewScheduler() + defer func() { _ = s.Shutdown() }() + + _, _ = s.NewJob( + gocron.CronJob( + "0 4 * * *", + false, + ), + gocron.NewTask( + func() { + repoService.ForceUpdate() + }, + ), + ) + taskController := taskcontroller.New( r.app, taskService, -- 2.45.2 From 73a654bfaaf24ee04894bc537f5556a8549ee995 Mon Sep 17 00:00:00 2001 From: Maxim Slipenko Date: Fri, 13 Dec 2024 15:01:49 +0300 Subject: [PATCH 6/7] wip --- .dockerignore | 3 +++ Dockerfile | 8 +++---- go.mod | 5 ++++ go.sum | 10 ++++++++ internal/controllers/taskcontroller/upload.go | 12 ---------- internal/services/reposervice/service.go | 24 +++++++++++++++++-- internal/services/taskservice/create.go | 12 +++++++--- internal/services/taskservice/service.go | 8 +++---- internal/services/taskservice/upload.go | 1 - 9 files changed, 57 insertions(+), 26 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..f1914ae --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +.git +app.db +uploads \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 545fbf9..b3fa0b2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,15 @@ # Первый этап: сборка приложения FROM golang:latest AS builder -WORKDIR /rest-app +WORKDIR /app COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN go build -o alt-storage +RUN go build -o aides-repo-api ./cmd/aides-repo-api/main.go # Второй этап: создание финального образа FROM registry.altlinux.org/alt/alt:sisyphus -COPY --from=builder /rest-app/alt-storage /bin/main -ENTRYPOINT ["/bin/main"] \ No newline at end of file +COPY --from=builder /app/aides-repo-api /bin/aides-repo-api +ENTRYPOINT ["/bin/aides-repo-api"] \ No newline at end of file diff --git a/go.mod b/go.mod index fcc8a8c..78aa012 100644 --- a/go.mod +++ b/go.mod @@ -10,9 +10,14 @@ require ( require ( github.com/ajg/form v1.5.1 // indirect + github.com/go-co-op/gocron/v2 v2.13.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/jonboulle/clockwork v0.4.0 // indirect github.com/mattn/go-sqlite3 v1.14.24 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/text v0.21.0 // indirect gorm.io/driver/sqlite v1.5.7 // indirect gorm.io/gorm v1.25.12 // indirect diff --git a/go.sum b/go.sum index 5acd7b7..1c90d0d 100644 --- a/go.sum +++ b/go.sum @@ -6,12 +6,22 @@ github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= github.com/go-chi/render v1.0.3/go.mod h1:/gr3hVkmYR0YlEy3LxCuVRFzEu9Ruok+gFqbIofjao0= +github.com/go-co-op/gocron/v2 v2.13.0 h1:iGU/RoZvf4GF5hIZUkDSFvvajk9K3W4YgocarBol/ME= +github.com/go-co-op/gocron/v2 v2.13.0/go.mod h1:ZF70ZwEqz0OO4RBXE1sNxnANy/zvwLcattWEFsqpKig= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= +golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I= diff --git a/internal/controllers/taskcontroller/upload.go b/internal/controllers/taskcontroller/upload.go index 9700716..d60e507 100644 --- a/internal/controllers/taskcontroller/upload.go +++ b/internal/controllers/taskcontroller/upload.go @@ -12,7 +12,6 @@ import ( type TaskUploadResponse struct { TaskID string `json:"taskID"` - Repo string `json:"repo"` StatusText string `json:"status"` } @@ -39,15 +38,6 @@ func (c *TaskController) Upload(w http.ResponseWriter, r *http.Request) { return } - repo := r.FormValue("repo") - if repo == "" { - render.Render(w, r, &errors.ErrResponse{ - HTTPStatusCode: http.StatusBadRequest, - StatusText: "Missing required repo field", - }) - return - } - files := r.MultipartForm.File["files"] for _, fileHeader := range files { if fileHeader.Size > (1024 << 20) { // Limit each file size to 10MB @@ -61,7 +51,6 @@ func (c *TaskController) Upload(w http.ResponseWriter, r *http.Request) { err = c.taskService.Upload(&taskservice.TaskUploadInput{ TaskID: taskID, - Repo: repo, Files: files, }) if err != nil { @@ -76,7 +65,6 @@ func (c *TaskController) Upload(w http.ResponseWriter, r *http.Request) { response := TaskUploadResponse{ TaskID: taskID, - Repo: repo, StatusText: "Success!", } diff --git a/internal/services/reposervice/service.go b/internal/services/reposervice/service.go index bac9519..311bfce 100644 --- a/internal/services/reposervice/service.go +++ b/internal/services/reposervice/service.go @@ -101,8 +101,6 @@ func (s *Service) ForceUpdate() { } for _, el := range tasks { - fmt.Println(el.Files) - taskPath := path.Join( s.app.Config.UploadDir, "tasks", strconv.FormatUint(uint64(el.ID), 10), ) @@ -131,4 +129,26 @@ func (s *Service) ForceUpdate() { Update( "current_task_id", gorm.Expr("last_task_id"), ) + + os.MkdirAll(path.Join(s.app.Config.UploadDir, "repo"), os.ModePerm) + + aPath := path.Join(s.app.Config.UploadDir, "future_repo", "Sisyphus") + bPath := path.Join(s.app.Config.UploadDir, "repo", "Sisyphus") + cPath := path.Join(s.app.Config.UploadDir, "repo", ".Sisyphus") + + if _, err := os.Stat(bPath); err == nil { + fmt.Printf("Moving %s to %s\n", bPath, cPath) + if err := os.Rename(bPath, cPath); err != nil { + } + } else if !os.IsNotExist(err) { + + } + + if err := os.Rename(aPath, bPath); err != nil { + panic(err) + } + + if err := os.RemoveAll(cPath); err != nil { + + } } diff --git a/internal/services/taskservice/create.go b/internal/services/taskservice/create.go index d178359..aa81c21 100644 --- a/internal/services/taskservice/create.go +++ b/internal/services/taskservice/create.go @@ -1,17 +1,23 @@ package taskservice -import "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" +import ( + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" +) func (s *Service) Create(repo string) (*models.Task, error) { taskRepo := models.GitRepo{ Name: repo, } - s.app.Db.FirstOrCreate(&taskRepo) + s.app.Db.Debug(). + Where(&taskRepo). + FirstOrCreate(&taskRepo) altRepo := models.ALTRepo{ Name: "Sisyphus", } - s.app.Db.FirstOrCreate(&altRepo) + s.app.Db. + Where(&altRepo). + FirstOrCreate(&altRepo) task := models.Task{ RepoID: taskRepo.ID, diff --git a/internal/services/taskservice/service.go b/internal/services/taskservice/service.go index c0e78f3..68d9511 100644 --- a/internal/services/taskservice/service.go +++ b/internal/services/taskservice/service.go @@ -26,11 +26,11 @@ func (s *Service) onTaskComplete(task *models.Task) error { grart := models.GitRepoAltRepoTask{ ALTRepoID: 1, - Repo: task.Repo, + RepoID: task.RepoID, } - s.app.Db.Debug().Where( - &grart, - ).FirstOrCreate(&grart) + s.app.Db.Debug(). + Where(&grart). + FirstOrCreate(&grart) grart.LastTaskID = task.ID diff --git a/internal/services/taskservice/upload.go b/internal/services/taskservice/upload.go index 0465c32..4334721 100644 --- a/internal/services/taskservice/upload.go +++ b/internal/services/taskservice/upload.go @@ -17,7 +17,6 @@ import ( type TaskUploadInput struct { TaskID string - Repo string Files []*multipart.FileHeader } -- 2.45.2 From e0c5a1049ea503159d47699d3e3686922e8047af Mon Sep 17 00:00:00 2001 From: Maxim Slipenko Date: Fri, 13 Dec 2024 23:39:41 +0300 Subject: [PATCH 7/7] wip --- .dockerignore | 3 ++- Dockerfile | 6 ++++++ go.mod | 7 +++++++ go.sum | 21 +++++++++++++++++++++ internal/app/app.go | 11 ++++++++--- internal/config/config.go | 6 ++++++ internal/models/db.go | 6 +++--- internal/services/reposervice/service.go | 2 ++ internal/services/taskservice/service.go | 10 +++++----- 9 files changed, 60 insertions(+), 12 deletions(-) diff --git a/.dockerignore b/.dockerignore index f1914ae..cb1343b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,4 @@ .git app.db -uploads \ No newline at end of file +uploads +Dockerfile \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index b3fa0b2..0e2f0ff 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,5 +11,11 @@ RUN go build -o aides-repo-api ./cmd/aides-repo-api/main.go # Второй этап: создание финального образа FROM registry.altlinux.org/alt/alt:sisyphus +RUN \ + apt-get update && \ + apt-get install apt-repo-tools && \ + rm -f /var/lib/apt/lists/*.* && \ + rm -f /var/cache/apt/*.bin + COPY --from=builder /app/aides-repo-api /bin/aides-repo-api ENTRYPOINT ["/bin/aides-repo-api"] \ No newline at end of file diff --git a/go.mod b/go.mod index 78aa012..e9427b5 100644 --- a/go.mod +++ b/go.mod @@ -12,13 +12,20 @@ require ( github.com/ajg/form v1.5.1 // indirect github.com/go-co-op/gocron/v2 v2.13.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/pgx/v5 v5.7.1 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect github.com/mattn/go-sqlite3 v1.14.24 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect + golang.org/x/sync v0.10.0 // indirect golang.org/x/text v0.21.0 // indirect + gorm.io/driver/postgres v1.5.11 // indirect gorm.io/driver/sqlite v1.5.7 // indirect gorm.io/gorm v1.25.12 // indirect ) diff --git a/go.sum b/go.sum index 1c90d0d..a7284fa 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/caarlos0/env/v11 v11.2.2 h1:95fApNrUyueipoZN/EhA8mMxiNxrBwDa+oAZrMWl3Kg= github.com/caarlos0/env/v11 v11.2.2/go.mod h1:JBfcdeQiBoI3Zh1QRAWfe+tpiNTmDtcCj/hHHHMx0vc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw= github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-chi/render v1.0.3 h1:AsXqd2a1/INaIfUSKq3G5uA8weYx20FOsM7uSoCyyt4= @@ -10,6 +11,14 @@ github.com/go-co-op/gocron/v2 v2.13.0 h1:iGU/RoZvf4GF5hIZUkDSFvvajk9K3W4YgocarBo github.com/go-co-op/gocron/v2 v2.13.0/go.mod h1:ZF70ZwEqz0OO4RBXE1sNxnANy/zvwLcattWEFsqpKig= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= +github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= @@ -18,12 +27,24 @@ github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.5.11 h1:ubBVAfbKEUld/twyKZ0IYn9rSQh448EdelLYk9Mv314= +gorm.io/driver/postgres v1.5.11/go.mod h1:DX3GReXH+3FPWGrrgffdvCk3DQ1dwDPdmbenSkweRGI= gorm.io/driver/sqlite v1.5.7 h1:8NvsrhP0ifM7LX9G4zPB97NwovUakUxc+2V2uuf3Z1I= gorm.io/driver/sqlite v1.5.7/go.mod h1:U+J8craQU6Fzkcvu8oLeAQmi50TkwPEhHDEjQZXDah4= gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= diff --git a/internal/app/app.go b/internal/app/app.go index 6f9878a..0b8d304 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1,9 +1,11 @@ package app import ( + "fmt" + "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config" "code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models" - "gorm.io/driver/sqlite" + "gorm.io/driver/postgres" "gorm.io/gorm" ) @@ -13,7 +15,10 @@ type App struct { } func New() (*App, error) { - db, err := gorm.Open(sqlite.Open("app.db"), &gorm.Config{}) + cfg := config.New() + dsn := fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=disable", + cfg.DBHost, cfg.DBPort, cfg.DBUser, cfg.DBPassword, cfg.DBName) + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{}) if err != nil { return nil, err } @@ -25,7 +30,7 @@ func New() (*App, error) { }) return &App{ - Config: config.New(), + Config: cfg, Db: db, }, nil } diff --git a/internal/config/config.go b/internal/config/config.go index 36a30ab..08ee12d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -11,6 +11,12 @@ type Config struct { UploadDir string `env:"UPLOAD_DIR" envDefault:"./uploads"` Port string `env:"PORT" envDefault:"8080"` MaxSizeUpload int64 `env:"MAX_SIZE_UPLOAD" envDefault:"104857600"` //100 MB + + DBHost string `env:"DB_HOST" envDefault:"localhost"` + DBPort string `env:"DB_PORT" envDefault:"5432"` + DBUser string `env:"DB_USER" envDefault:"user"` + DBPassword string `env:"DB_PASSWORD" envDefault:"password"` + DBName string `env:"DB_NAME" envDefault:"app_db"` } func New() *Config { diff --git a/internal/models/db.go b/internal/models/db.go index 46a995a..0def57a 100644 --- a/internal/models/db.go +++ b/internal/models/db.go @@ -69,9 +69,9 @@ type GitRepoAltRepoTask struct { ALTRepoID uint `gorm:"uniqueIndex:idx_gr_ar_gitaltrepotask"` ALTRepo *ALTRepo - LastTaskID uint + LastTaskID *uint LastTask *Task - CurrentTaskID uint - CurrentTask Task + CurrentTaskID *uint + CurrentTask *Task } diff --git a/internal/services/reposervice/service.go b/internal/services/reposervice/service.go index 311bfce..fa48962 100644 --- a/internal/services/reposervice/service.go +++ b/internal/services/reposervice/service.go @@ -151,4 +151,6 @@ func (s *Service) ForceUpdate() { if err := os.RemoveAll(cPath); err != nil { } + + os.RemoveAll(path.Join(s.app.Config.UploadDir, "future_repo")) } diff --git a/internal/services/taskservice/service.go b/internal/services/taskservice/service.go index 68d9511..7191eef 100644 --- a/internal/services/taskservice/service.go +++ b/internal/services/taskservice/service.go @@ -32,7 +32,7 @@ func (s *Service) onTaskComplete(task *models.Task) error { Where(&grart). FirstOrCreate(&grart) - grart.LastTaskID = task.ID + grart.LastTaskID = &task.ID if err := s.app.Db.Save(&grart).Error; err != nil { return err @@ -45,11 +45,11 @@ func (s *Service) onTaskComplete(task *models.Task) error { func (s *Service) tasksCleanup(r *models.GitRepoAltRepoTask, N int) { excludedTaskIDs := []uint{} - if r.CurrentTaskID != 0 { - excludedTaskIDs = append(excludedTaskIDs, r.CurrentTaskID) + if r.CurrentTaskID != nil { + excludedTaskIDs = append(excludedTaskIDs, *r.CurrentTaskID) } - if r.LastTaskID != 0 { - excludedTaskIDs = append(excludedTaskIDs, r.LastTaskID) + if r.LastTaskID != nil { + excludedTaskIDs = append(excludedTaskIDs, *r.LastTaskID) } var lastNTaskIDs []uint -- 2.45.2