epic: add more formatters, logger

This commit is contained in:
Максим Слипенко 2024-12-15 11:39:18 +03:00
parent 3438637c1e
commit 026e981415
15 changed files with 435 additions and 92 deletions

41
.golangci.yml Normal file
View file

@ -0,0 +1,41 @@
run:
timeout: 5m
exclude-dirs:
- "vendor"
exclue-files:
- ".*\\.gen\\.go"
linters-settings:
errcheck:
exclude-functions:
- github.com/go-chi/render.Render
goimports:
local-prefixes: "code.alt-gnome.ru/aides-infra/aides-repo-api"
gofmt:
simplify: true
gofumpt:
extra-rules: true
forbidigo:
forbid:
- p: ^fmt\.Print.*$
msg: Do not commit print statements.
linters:
enable:
- gofmt
- gofumpt
- goimports
- gocritic
- govet
- staticcheck
- unused
- errcheck
- typecheck
- forbidigo
issues:
fix: true
exclude-rules:
- path: _test\.go
linters:
- errcheck

View file

@ -1,14 +1,13 @@
GOFUMPT := go run mvdan.cc/gofumpt@v0.7.0
GOIMPORTS := go run golang.org/x/tools/cmd/goimports@v0.28.0
GCI := go run github.com/daixiang0/gci@v0.13.5
GOLINES := go run github.com/segmentio/golines@v0.12.2
GOLANGCI_LINT := go run github.com/golangci/golangci-lint/cmd/golangci-lint@v1.62.2
SWAG := go run github.com/swaggo/swag/cmd/swag@v1.16.4
format:
@echo "🛠️ Format code"
$(GOIMPORTS) -w .
$(GCI) write -s standard -s default -s "prefix(code.alt-gnome.ru/aides-infra/aides-repo-api)" .
$(GOLINES) -w .
$(GOFUMPT) -w .
@echo "✅ Format done."
@echo "🛠️ Format and Lint code with golangci-lint"
$(GOLANGCI_LINT) run --fix
$(SWAG) fmt
@echo "✅ Format and Lint done."
.PHONY: format
swag:
$(SWAG) init -g cmd/aides-repo-api/main.go
.PHONY: format swag

View file

@ -60,6 +60,57 @@ const docTemplate = `{
}
}
}
},
"/tasks/{taskID}/upload": {
"post": {
"description": "Upload multiple files associated with a specific task ID. Each file must be less than 10MB.",
"consumes": [
"multipart/form-data"
],
"produces": [
"application/json"
],
"tags": [
"tasks"
],
"summary": "Upload files to a task",
"parameters": [
{
"type": "string",
"description": "Task ID",
"name": "taskID",
"in": "path",
"required": true
},
{
"type": "file",
"description": "Files to upload",
"name": "files",
"in": "formData",
"required": true
}
],
"responses": {
"200": {
"description": "Successful file upload",
"schema": {
"$ref": "#/definitions/taskcontroller.TaskUploadResponse"
}
},
"400": {
"description": "Bad Request or File too large",
"schema": {
"$ref": "#/definitions/errors.ErrResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/errors.ErrResponse"
}
}
}
}
}
},
"definitions": {
@ -122,6 +173,17 @@ const docTemplate = `{
"type": "integer"
}
}
},
"taskcontroller.TaskUploadResponse": {
"type": "object",
"properties": {
"status": {
"type": "string"
},
"taskID": {
"type": "string"
}
}
}
}
}`

View file

@ -49,6 +49,57 @@
}
}
}
},
"/tasks/{taskID}/upload": {
"post": {
"description": "Upload multiple files associated with a specific task ID. Each file must be less than 10MB.",
"consumes": [
"multipart/form-data"
],
"produces": [
"application/json"
],
"tags": [
"tasks"
],
"summary": "Upload files to a task",
"parameters": [
{
"type": "string",
"description": "Task ID",
"name": "taskID",
"in": "path",
"required": true
},
{
"type": "file",
"description": "Files to upload",
"name": "files",
"in": "formData",
"required": true
}
],
"responses": {
"200": {
"description": "Successful file upload",
"schema": {
"$ref": "#/definitions/taskcontroller.TaskUploadResponse"
}
},
"400": {
"description": "Bad Request or File too large",
"schema": {
"$ref": "#/definitions/errors.ErrResponse"
}
},
"500": {
"description": "Internal Server Error",
"schema": {
"$ref": "#/definitions/errors.ErrResponse"
}
}
}
}
}
},
"definitions": {
@ -111,6 +162,17 @@
"type": "integer"
}
}
},
"taskcontroller.TaskUploadResponse": {
"type": "object",
"properties": {
"status": {
"type": "string"
},
"taskID": {
"type": "string"
}
}
}
}
}

View file

@ -43,6 +43,13 @@ definitions:
taskID:
type: integer
type: object
taskcontroller.TaskUploadResponse:
properties:
status:
type: string
taskID:
type: string
type: object
info:
contact: {}
paths:
@ -76,4 +83,39 @@ paths:
summary: Create a new task
tags:
- Tasks
/tasks/{taskID}/upload:
post:
consumes:
- multipart/form-data
description: Upload multiple files associated with a specific task ID. Each
file must be less than 10MB.
parameters:
- description: Task ID
in: path
name: taskID
required: true
type: string
- description: Files to upload
in: formData
name: files
required: true
type: file
produces:
- application/json
responses:
"200":
description: Successful file upload
schema:
$ref: '#/definitions/taskcontroller.TaskUploadResponse'
"400":
description: Bad Request or File too large
schema:
$ref: '#/definitions/errors.ErrResponse'
"500":
description: Internal Server Error
schema:
$ref: '#/definitions/errors.ErrResponse'
summary: Upload files to a task
tags:
- tasks
swagger: "2.0"

View file

@ -3,16 +3,14 @@ package app
import (
"fmt"
"net/http"
"os"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"moul.io/zapgorm2"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/config"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/controllers/taskcontroller"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/logger"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/router"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/cronservice"
@ -21,7 +19,7 @@ import (
)
type App struct {
logger *zap.Logger
logger *logger.ZapLogger
db *gorm.DB
config *config.Config
@ -49,25 +47,11 @@ func New() (*App, error) {
}
func (app *App) createLogger() {
atomic := zap.NewAtomicLevel()
atomic.SetLevel(zapcore.DebugLevel)
enconfig := zap.NewProductionEncoderConfig()
enconfig.EncodeTime = zapcore.ISO8601TimeEncoder
enconfig.TimeKey = "timestamp"
logger := zap.New(zapcore.NewCore(
zapcore.NewJSONEncoder(enconfig),
zapcore.Lock(os.Stdout),
atomic,
))
app.logger = logger
app.logger = logger.GetLogger()
}
func (app *App) createDb() {
logger := zapgorm2.New(app.logger)
logger := zapgorm2.New(app.logger.GetZap())
logger.SetAsDefault()
dsn := fmt.Sprintf(
"host=%s port=%s user=%s password=%s dbname=%s sslmode=disable",
@ -81,7 +65,10 @@ func (app *App) createDb() {
if err != nil {
panic(err)
}
db.AutoMigrate(&models.Task{}, &models.GitRepoAltRepoTask{}, &models.RPMFile{})
err = db.AutoMigrate(&models.Task{}, &models.GitRepoAltRepoTask{}, &models.RPMFile{})
if err != nil {
panic(err)
}
db.FirstOrCreate(&models.ALTRepo{
Name: "Sisyphus",
})
@ -127,11 +114,13 @@ func (app *App) Init() {
func (app *App) Run() {
app.logger.Info(
"Сервер запущен",
zap.Int(
"port",
app.config.Port,
),
map[string]interface{}{
"port": app.config.Port,
},
)
http.ListenAndServe(fmt.Sprintf(":%d", app.config.Port), app.router.Setup())
err := http.ListenAndServe(fmt.Sprintf(":%d", app.config.Port), app.router.Setup())
if err != nil {
panic(err)
}
}

View file

@ -23,6 +23,8 @@ func (c *CreateTaskResponse) Render(w http.ResponseWriter, r *http.Request) erro
return nil
}
// Create a new task
//
// @Summary Create a new task
// @Description Create a new task for a specific repository
// @Tags Tasks

View file

@ -1,13 +1,13 @@
package taskcontroller
import (
"fmt"
"net/http"
"github.com/go-chi/chi/v5"
"github.com/go-chi/render"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/common/errors"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/logger"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/services/taskservice"
)
@ -20,6 +20,19 @@ func (rd *TaskUploadResponse) Render(w http.ResponseWriter, r *http.Request) err
return nil
}
// Upload handles file uploads for a specific task.
//
// @Summary Upload files to a task
// @Description Upload multiple files associated with a specific task ID. Each file must be less than 10MB.
// @Tags tasks
// @Accept multipart/form-data
// @Produce json
// @Param taskID path string true "Task ID"
// @Param files formData file true "Files to upload"
// @Success 200 {object} TaskUploadResponse "Successful file upload"
// @Failure 400 {object} errors.ErrResponse "Bad Request or File too large"
// @Failure 500 {object} errors.ErrResponse "Internal Server Error"
// @Router /tasks/{taskID}/upload [post]
func (c *TaskController) Upload(w http.ResponseWriter, r *http.Request) {
taskID := chi.URLParam(r, "taskID")
if taskID == "" {
@ -41,7 +54,7 @@ func (c *TaskController) Upload(w http.ResponseWriter, r *http.Request) {
files := r.MultipartForm.File["files"]
for _, fileHeader := range files {
if fileHeader.Size > (1024 << 20) { // Limit each file size to 10MB
if fileHeader.Size > (1024 << 20) {
render.Render(w, r, &errors.ErrResponse{
HTTPStatusCode: http.StatusBadRequest,
StatusText: "File too large",
@ -55,7 +68,10 @@ func (c *TaskController) Upload(w http.ResponseWriter, r *http.Request) {
Files: files,
})
if err != nil {
fmt.Println(err)
log := logger.GetLogger()
log.Error("Error while upload task", map[string]interface{}{
"err": err,
})
render.Render(w, r, &errors.ErrResponse{
HTTPStatusCode: http.StatusInternalServerError,
StatusText: "Internal Server Error",

26
internal/logger/logger.go Normal file
View file

@ -0,0 +1,26 @@
package logger
import (
"sync"
)
var (
instance *ZapLogger
once sync.Once
)
// Logger определяет интерфейс для логгера
type Logger interface {
Debug(msg string, fields map[string]interface{})
Info(msg string, fields map[string]interface{})
Warn(msg string, fields map[string]interface{})
Error(msg string, fields map[string]interface{})
Fatal(msg string, fields map[string]interface{})
}
func GetLogger() *ZapLogger {
once.Do(func() {
instance = NewZapLogger()
})
return instance
}

View file

@ -0,0 +1,94 @@
package logger
import (
"os"
"go.uber.org/zap"
"go.uber.org/zap/zapcore"
)
type ZapLogger struct {
logger *zap.Logger
}
func NewZapLogger() *ZapLogger {
atomicLevel := zap.NewAtomicLevel()
atomicLevel.SetLevel(zapcore.DebugLevel)
encoderConfig := zap.NewProductionEncoderConfig()
encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder
encoderConfig.TimeKey = "timestamp"
core := zapcore.NewCore(
zapcore.NewJSONEncoder(encoderConfig),
zapcore.Lock(os.Stdout),
atomicLevel,
)
return &ZapLogger{
logger: zap.New(core),
}
}
func (l *ZapLogger) GetZap() *zap.Logger {
return l.logger
}
func (l *ZapLogger) Debug(msg string, fields ...map[string]interface{}) {
l.log(zap.DebugLevel, msg, fields...)
}
func (l *ZapLogger) Info(msg string, fields ...map[string]interface{}) {
l.log(zap.InfoLevel, msg, fields...)
}
func (l *ZapLogger) Warn(msg string, fields ...map[string]interface{}) {
l.log(zap.WarnLevel, msg, fields...)
}
func (l *ZapLogger) Error(msg string, fields ...map[string]interface{}) {
l.log(zap.ErrorLevel, msg, fields...)
}
func (l *ZapLogger) Fatal(msg string, fields ...map[string]interface{}) {
l.log(zap.FatalLevel, msg, fields...)
}
func (l *ZapLogger) log(level zapcore.Level, msg string, fields ...map[string]interface{}) {
if len(fields) == 0 || fields[0] == nil {
switch level {
case zap.DebugLevel:
l.logger.Debug(msg)
case zap.InfoLevel:
l.logger.Info(msg)
case zap.WarnLevel:
l.logger.Warn(msg)
case zap.ErrorLevel:
l.logger.Error(msg)
case zap.FatalLevel:
l.logger.Fatal(msg)
}
} else {
zapFields := convertMapToZapFields(fields[0])
switch level {
case zap.DebugLevel:
l.logger.Debug(msg, zapFields...)
case zap.InfoLevel:
l.logger.Info(msg, zapFields...)
case zap.WarnLevel:
l.logger.Warn(msg, zapFields...)
case zap.ErrorLevel:
l.logger.Error(msg, zapFields...)
case zap.FatalLevel:
l.logger.Fatal(msg, zapFields...)
}
}
}
func convertMapToZapFields(fields map[string]interface{}) []zap.Field {
zapFields := make([]zap.Field, 0, len(fields))
for key, value := range fields {
zapFields = append(zapFields, zap.Any(key, value))
}
return zapFields
}

View file

@ -1,11 +1,12 @@
package cronservice
import (
"fmt"
"log"
"time"
"github.com/go-co-op/gocron/v2"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/logger"
)
type RepoService interface {
@ -34,7 +35,10 @@ func (s *Service) SetupCronJobs() {
_, err := s.scheduler.NewJob(
gocron.CronJob("* * * * *", false),
gocron.NewTask(func() {
fmt.Println("Cron run!")
log := logger.GetLogger()
log.Info(
"Cron run!",
)
s.repoService.ForceUpdate()
}),
)

View file

@ -9,6 +9,7 @@ import (
"gorm.io/gorm"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/logger"
"code.alt-gnome.ru/aides-infra/aides-repo-api/internal/models"
)
@ -43,6 +44,7 @@ func createSymlink(target, link string) error {
}
func runGenbasedir(repoDir, arch, repoName string) {
log := logger.GetLogger()
cmd := exec.Command(
"genbasedir",
"--bloat",
@ -55,18 +57,30 @@ func runGenbasedir(repoDir, arch, repoName string) {
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
fmt.Printf("Failed to run genbasedir for %s: %v\n", arch, err)
log.Error(
"Failed to run genbasedir",
map[string]interface{}{
"arch": arch,
"error": err,
},
)
os.Exit(1)
}
fmt.Printf("Successfully ran genbasedir for %s\n", arch)
log.Info("Successfully ran genbasedir", map[string]interface{}{
"arch": arch,
})
}
func createRepoDirs(repoDir, repoName, arch string) {
log := logger.GetLogger()
// Create the 'base' directory
baseDir := path.Join(repoDir, arch, "base")
err := os.MkdirAll(baseDir, os.ModePerm)
if err != nil {
fmt.Printf("Failed to create directory %s: %v\n", baseDir, err)
log.Error("Failed to create directory", map[string]interface{}{
"directory": baseDir,
"err": err,
})
os.Exit(1)
}
@ -74,7 +88,10 @@ func createRepoDirs(repoDir, repoName, arch string) {
rpmsDir := path.Join(repoDir, arch, fmt.Sprintf("RPMS.%s", repoName))
err = os.MkdirAll(rpmsDir, os.ModePerm)
if err != nil {
fmt.Printf("Failed to create directory %s: %v\n", rpmsDir, err)
log.Error("Failed to create directory", map[string]interface{}{
"directory": rpmsDir,
"err": err,
})
os.Exit(1)
}
}
@ -101,10 +118,13 @@ func (s *Service) ForceUpdate() {
repoPath := path.Join(s.config.GetUploadDir(), "future_repo", "Sisyphus")
os.MkdirAll(
err := os.MkdirAll(
repoPath,
os.ModePerm,
)
if err != nil {
panic(err)
}
repoName := "aides"
architectures := []string{"x86_64", "noarch"}
@ -114,10 +134,6 @@ func (s *Service) ForceUpdate() {
}
for _, el := range tasks {
taskPath := path.Join(
s.config.GetUploadDir(), "tasks", strconv.FormatUint(uint64(el.ID), 10),
)
for _, fileInfo := range el.Files {
localFilePath := path.Join(
strconv.FormatUint(uint64(el.ID), 10), fileInfo.Name,
@ -131,9 +147,10 @@ func (s *Service) ForceUpdate() {
fileInfo.Name,
)
targetPath := path.Join("../../../../tasks/", localFilePath)
createSymlink(targetPath, symLink)
fmt.Println(path.Join(taskPath, fileInfo.Name))
err := createSymlink(targetPath, symLink)
if err != nil {
panic(err)
}
}
}
@ -150,17 +167,21 @@ func (s *Service) ForceUpdate() {
"current_task_id", gorm.Expr("last_task_id"),
)
os.MkdirAll(path.Join(s.config.GetUploadDir(), "repo"), os.ModePerm)
err = os.MkdirAll(path.Join(s.config.GetUploadDir(), "repo"), os.ModePerm)
if err != nil {
panic(err)
}
aPath := path.Join(s.config.GetUploadDir(), "future_repo", "Sisyphus")
bPath := path.Join(s.config.GetUploadDir(), "repo", "Sisyphus")
cPath := path.Join(s.config.GetUploadDir(), "repo", ".Sisyphus")
if _, err := os.Stat(bPath); err == nil {
fmt.Printf("Moving %s to %s\n", bPath, cPath)
if err := os.Rename(bPath, cPath); err != nil {
panic(err)
}
} else if !os.IsNotExist(err) {
panic(err)
}
if err := os.Rename(aPath, bPath); err != nil {
@ -168,7 +189,10 @@ func (s *Service) ForceUpdate() {
}
if err := os.RemoveAll(cPath); err != nil {
panic(err)
}
os.RemoveAll(path.Join(s.config.GetUploadDir(), "future_repo"))
if err := os.RemoveAll(path.Join(s.config.GetUploadDir(), "future_repo")); err != nil {
panic(err)
}
}

View file

@ -50,7 +50,7 @@ func (s *Service) onTaskComplete(task *models.Task) error {
return nil
}
func (s *Service) tasksCleanup(r *models.GitRepoAltRepoTask, N int) {
func (s *Service) tasksCleanup(r *models.GitRepoAltRepoTask, n int) {
excludedTaskIDs := []uint{}
if r.CurrentTaskID != nil {
excludedTaskIDs = append(excludedTaskIDs, *r.CurrentTaskID)
@ -68,7 +68,7 @@ func (s *Service) tasksCleanup(r *models.GitRepoAltRepoTask, N int) {
Where("status = ?", models.StatusCompleted).
Where("id NOT IN ?", excludedTaskIDs).
Order("created_at DESC").
Limit(N).
Limit(n).
Pluck("id", &lastNTaskIDs)
excludedTaskIDs = append(excludedTaskIDs, lastNTaskIDs...)

View file

@ -58,7 +58,10 @@ func (s *Service) Upload(input *TaskUploadInput) error {
localPath := path.Join(input.TaskID)
taskFolderPath := path.Join(s.config.GetUploadDir(), "tasks", localPath)
os.MkdirAll(taskFolderPath, os.ModePerm)
err = os.MkdirAll(taskFolderPath, os.ModePerm)
if err != nil {
return err
}
for _, fileHeader := range files {
file, err := fileHeader.Open()
@ -121,7 +124,6 @@ func (s *Service) Upload(input *TaskUploadInput) error {
}
task.Status = models.StatusCompleted
s.onTaskComplete(&task)
return nil
return s.onTaskComplete(&task)
}

View file

@ -1,20 +0,0 @@
package taskservice
import (
"fmt"
"os"
)
func createSymlink(target, link string) error {
if _, err := os.Lstat(link); err == nil {
if err := os.Remove(link); err != nil {
return fmt.Errorf("failed to remove existing file or symlink: %w", err)
}
}
if err := os.Symlink(target, link); err != nil {
return fmt.Errorf("failed to create symlink: %w", err)
}
return nil
}