feat: add apt archives caching
This commit is contained in:
parent
2c156c1c3f
commit
6c66e8d10f
8 changed files with 136 additions and 286 deletions
|
|
@ -12,7 +12,7 @@ BUILD_COMMAND = (
|
|||
)
|
||||
|
||||
|
||||
@app.command()
|
||||
@app.command(help="Build package from spec in isolated environment")
|
||||
def build(
|
||||
no_cache: bool = typer.Option(False, "--no-cache", help="Disable cache")
|
||||
):
|
||||
|
|
@ -51,6 +51,11 @@ def build(
|
|||
os.makedirs(alr_cache, exist_ok=True)
|
||||
command.extend(["-v", f"{alr_cache}:/home/buildbot/.cache/alr"])
|
||||
|
||||
apt_archives = os.path.join(cache_dir, "apt_archives")
|
||||
os.makedirs(apt_archives, exist_ok=True)
|
||||
os.makedirs(os.path.join(apt_archives, "partial"), exist_ok=True)
|
||||
command.extend(["-v", f"{apt_archives}:/var/cache/apt/archives"])
|
||||
|
||||
command.extend([IMAGE, "/bin/sh", "-c", BUILD_COMMAND])
|
||||
|
||||
print(" ".join(command))
|
||||
|
|
|
|||
|
|
@ -68,6 +68,6 @@ def update_checksums(script_path):
|
|||
print(new_checksums_block)
|
||||
|
||||
|
||||
@app.command()
|
||||
@app.command(help="Get checksums for new sources")
|
||||
def checksums():
|
||||
update_checksums(f"{os.getcwd()}/alr.sh")
|
||||
|
|
|
|||
|
|
@ -5,35 +5,69 @@ import typer
|
|||
|
||||
from aides_spec.utils.empty_template import create_from_empty_template
|
||||
from aides_spec.utils.from_pkgbuild import (
|
||||
PkgbuildDownloader,
|
||||
create_from_pkgbuild,
|
||||
download_pkgbuild,
|
||||
)
|
||||
|
||||
app = typer.Typer()
|
||||
|
||||
|
||||
@app.command()
|
||||
def create(
|
||||
from_aur: Annotated[Optional[str], typer.Option()] = None,
|
||||
from_pkgbuild: Annotated[Optional[str], typer.Option()] = None,
|
||||
empty_template: Annotated[Optional[bool], typer.Option()] = None,
|
||||
):
|
||||
output_file = "alr.sh"
|
||||
|
||||
if empty_template:
|
||||
def process_empty_template(output_file: str):
|
||||
"""Handles creation from an empty template."""
|
||||
typer.echo("Creating spec from an empty template...")
|
||||
create_from_empty_template(output_file)
|
||||
elif from_aur:
|
||||
print(f"Загружаем PKGBUILD для пакета '{from_aur}' из AUR...")
|
||||
content = download_pkgbuild(from_aur)
|
||||
create_from_pkgbuild(content, output_file)
|
||||
elif from_pkgbuild:
|
||||
print(f"Читаем PKGBUILD из локального файла '{from_pkgbuild}'...")
|
||||
|
||||
|
||||
def process_from_aur(package_name: str, output_file: str):
|
||||
"""Handles creation from an AUR package."""
|
||||
typer.echo(
|
||||
f"Downloading PKGBUILD for package '{package_name}' from AUR..."
|
||||
)
|
||||
try:
|
||||
with open(from_pkgbuild, "rb") as f:
|
||||
content = PkgbuildDownloader.download_and_extract(package_name)
|
||||
create_from_pkgbuild(content, output_file)
|
||||
except Exception as e:
|
||||
typer.echo(
|
||||
f"Error downloading PKGBUILD for '{package_name}': {e}", err=True
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def process_from_pkgbuild(file_path: str, output_file: str):
|
||||
"""Handles creation from a local PKGBUILD file."""
|
||||
typer.echo(f"Reading PKGBUILD from local file '{file_path}'...")
|
||||
try:
|
||||
with open(file_path, "rb") as f:
|
||||
content = f.read()
|
||||
create_from_pkgbuild(content, output_file)
|
||||
except IOError as e:
|
||||
print(f"Ошибка чтения файла '{from_pkgbuild}': {e}")
|
||||
typer.echo(f"Error reading file '{file_path}': {e}", err=True)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
@app.command(help="Create spec (empty, from PKGBUILD or AUR)")
|
||||
def create(
|
||||
from_aur: Annotated[
|
||||
Optional[str], typer.Option(help="Package name to fetch from AUR")
|
||||
] = None,
|
||||
from_pkgbuild: Annotated[
|
||||
Optional[str], typer.Option(help="Path to local PKGBUILD file")
|
||||
] = None,
|
||||
empty_template: Annotated[
|
||||
Optional[bool], typer.Option(help="Create spec from an empty template")
|
||||
] = None,
|
||||
):
|
||||
"""Main function to handle spec creation."""
|
||||
output_file = "alr.sh"
|
||||
|
||||
if empty_template:
|
||||
process_empty_template(output_file)
|
||||
elif from_aur:
|
||||
process_from_aur(from_aur, output_file)
|
||||
elif from_pkgbuild:
|
||||
process_from_pkgbuild(from_pkgbuild, output_file)
|
||||
else:
|
||||
typer.echo(
|
||||
"No valid option provided. Use --help for usage details.", err=True
|
||||
)
|
||||
sys.exit(1)
|
||||
|
|
|
|||
|
|
@ -7,24 +7,24 @@ if TYPE_CHECKING:
|
|||
|
||||
|
||||
class Replaces(TypedDict):
|
||||
node: Node
|
||||
node: "Node"
|
||||
content: str
|
||||
|
||||
|
||||
class Appends(TypedDict):
|
||||
node: Node
|
||||
node: "Node"
|
||||
content: str
|
||||
|
||||
|
||||
class BaseReplacer:
|
||||
def __init__(self, content, tree: Tree, ctx: dict | None = None):
|
||||
def __init__(self, content, tree: "Tree", ctx: dict | None = None):
|
||||
self.content = content
|
||||
self.tree = tree
|
||||
self.replaces: List[Replaces] = []
|
||||
self.appends: List[Appends] = []
|
||||
self.ctx = ctx
|
||||
|
||||
def _node_text(self, node: Node):
|
||||
def _node_text(self, node: "Node"):
|
||||
"""Helper function to get the text of a node."""
|
||||
return self.content[node.start_byte : node.end_byte].decode("utf-8")
|
||||
|
||||
|
|
|
|||
|
|
@ -1,91 +0,0 @@
|
|||
from aides_spec.replacers.arch_replacer import ArchReplacer
|
||||
from aides_spec.replacers.base import BaseReplacer
|
||||
|
||||
|
||||
class ChecksumsReplacer(BaseReplacer):
|
||||
CHECKSUMS_REPLACEMENTS = {
|
||||
"b2sums": "blake2b-512",
|
||||
"sha512sums": "sha512",
|
||||
"sha384sums": "sha384",
|
||||
"sha256sums": "sha256",
|
||||
"sha224sums": "sha224",
|
||||
"sha1sums": "sha1",
|
||||
"md5sums": "md5",
|
||||
}
|
||||
|
||||
def process(self):
|
||||
root_node = self.tree.root_node
|
||||
|
||||
sums = self.CHECKSUMS_REPLACEMENTS.keys()
|
||||
arches = ArchReplacer.ARCH_MAPPING.keys()
|
||||
|
||||
checksums = dict()
|
||||
combinations = {(s, a) for s in sums for a in arches}.union(
|
||||
{(s, None) for s in sums}
|
||||
)
|
||||
|
||||
def find_replacements(node):
|
||||
if node.type == "variable_assignment":
|
||||
var_node = node.child_by_field_name("name")
|
||||
value_node = node.child_by_field_name("value")
|
||||
|
||||
if var_node and value_node:
|
||||
var_name = self._node_text(var_node)
|
||||
for sum_part, arch_part in combinations:
|
||||
if (
|
||||
sum_part == var_name
|
||||
if arch_part is None
|
||||
else f"{sum_part}_{arch_part}" == var_name
|
||||
):
|
||||
checksums[(sum_part, arch_part)] = []
|
||||
for item in value_node.children:
|
||||
if item.type == "raw_string":
|
||||
element_text = self._node_text(item)
|
||||
if (
|
||||
element_text.startswith("'")
|
||||
and element_text.endswith("'")
|
||||
) or (
|
||||
element_text.startswith('"')
|
||||
and element_text.endswith('"')
|
||||
):
|
||||
# quote_char = element_text[0]
|
||||
hash = element_text[1:-1]
|
||||
else:
|
||||
hash = element_text
|
||||
|
||||
chcksm = self.CHECKSUMS_REPLACEMENTS[
|
||||
sum_part
|
||||
]
|
||||
|
||||
checksums[(sum_part, arch_part)].append(
|
||||
f"{chcksm}:{hash}"
|
||||
)
|
||||
self.replaces.append({"node": node, "content": ""})
|
||||
|
||||
for child in node.children:
|
||||
find_replacements(child)
|
||||
|
||||
find_replacements(root_node)
|
||||
|
||||
result = dict()
|
||||
|
||||
content = ""
|
||||
|
||||
for (sum_part, arch_part), hashes in checksums.items():
|
||||
key = (
|
||||
f"checksums_{ArchReplacer.ARCH_MAPPING[arch_part]}"
|
||||
if arch_part
|
||||
else "checksums"
|
||||
)
|
||||
result.setdefault(key, []).extend(hashes)
|
||||
|
||||
for key, value in result.items():
|
||||
content += f"""{key}=(
|
||||
'{"',\n '".join(value)}'
|
||||
)"""
|
||||
|
||||
self.appends.append({"node": root_node, "content": content})
|
||||
|
||||
print(result)
|
||||
|
||||
return self._apply_replacements()
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
from typing import TYPE_CHECKING
|
||||
|
||||
from aides_spec.replacers.base import BaseReplacer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tree_sitter import Node
|
||||
|
||||
|
||||
class LocalSourcesReplacer(BaseReplacer):
|
||||
def process(self):
|
||||
root_node = self.tree.root_node
|
||||
|
||||
self.local_files = []
|
||||
self.prepare_func_body = None
|
||||
|
||||
def find_replacements(node: Node):
|
||||
if node.type == "function_definition":
|
||||
func_name = self._node_text(node.child_by_field_name("name"))
|
||||
|
||||
if func_name == "prepare":
|
||||
self.prepare_func_body = node.child_by_field_name("body")
|
||||
|
||||
if node.type == "variable_assignment":
|
||||
var_node = node.child_by_field_name("name")
|
||||
value_node = node.child_by_field_name("value")
|
||||
|
||||
if var_node and value_node:
|
||||
var_name = self._node_text(var_node)
|
||||
if var_name == "sources":
|
||||
self._remove_local_files(value_node)
|
||||
|
||||
for child in node.children:
|
||||
find_replacements(child)
|
||||
|
||||
find_replacements(root_node)
|
||||
|
||||
copy_commands = "\n ".join(
|
||||
f'cp "${{scriptdir}}/{file}" "${{srcdir}}"'
|
||||
for file in self.local_files
|
||||
)
|
||||
|
||||
prepare_func_content = f"""
|
||||
{copy_commands}
|
||||
"""
|
||||
|
||||
print(self.local_files)
|
||||
|
||||
if self.prepare_func_body is not None:
|
||||
text = self._node_text(self.prepare_func_body)
|
||||
closing_brace_index = text.rfind("}")
|
||||
text = (
|
||||
text[:closing_brace_index]
|
||||
+ prepare_func_content
|
||||
+ text[closing_brace_index:]
|
||||
)
|
||||
self.replaces.append(
|
||||
{
|
||||
"node": self.prepare_func_body,
|
||||
"content": text,
|
||||
}
|
||||
)
|
||||
else:
|
||||
text = self._node_text(root_node)
|
||||
text = f"""prepare() {{
|
||||
{prepare_func_content}}}
|
||||
"""
|
||||
self.appends.append({"node": root_node, "content": text})
|
||||
|
||||
return self._apply_replacements()
|
||||
|
||||
def _remove_local_files(self, source_node: Node):
|
||||
updated_items = []
|
||||
for item_node in source_node.children:
|
||||
item_text = self._node_text(item_node)
|
||||
|
||||
if item_text == "(" or item_text == ")":
|
||||
continue
|
||||
|
||||
if "://" in item_text:
|
||||
updated_items.append(item_text)
|
||||
else:
|
||||
text = item_text
|
||||
if item_node.type == "string":
|
||||
text = self._node_text(item_node.child(1))
|
||||
|
||||
self.local_files.append(text)
|
||||
|
||||
new_content = "(\n " + " \n".join(updated_items) + "\n)"
|
||||
self.replaces.append(
|
||||
{
|
||||
"node": source_node,
|
||||
"content": new_content,
|
||||
}
|
||||
)
|
||||
|
|
@ -9,7 +9,7 @@ from aides_spec.replacers.base import BaseReplacer
|
|||
|
||||
|
||||
class StringValue(str):
|
||||
def __init__(self, node: Node):
|
||||
def __init__(self, node: "Node"):
|
||||
self.node = node
|
||||
|
||||
def get_text_value(self) -> str:
|
||||
|
|
@ -28,7 +28,7 @@ class StringValue(str):
|
|||
|
||||
|
||||
class Utils:
|
||||
def parse_variable_assignment(node: Node):
|
||||
def parse_variable_assignment(node: "Node"):
|
||||
var_node = node.child_by_field_name("name")
|
||||
value_node = node.child_by_field_name("value")
|
||||
|
||||
|
|
@ -37,7 +37,7 @@ class Utils:
|
|||
|
||||
return (var_node, value_node)
|
||||
|
||||
def get_string_values_from_array(node: Node):
|
||||
def get_string_values_from_array(node: "Node"):
|
||||
arr = []
|
||||
for item in node.children:
|
||||
if (
|
||||
|
|
@ -79,7 +79,7 @@ class SourcesReplacer(BaseReplacer):
|
|||
sources = dict()
|
||||
checksums = dict()
|
||||
|
||||
def execute(node: Node):
|
||||
def execute(node: "Node"):
|
||||
if node.type == "function_definition":
|
||||
func_name = self._node_text(node.child_by_field_name("name"))
|
||||
if func_name != "prepare":
|
||||
|
|
@ -116,7 +116,7 @@ class SourcesReplacer(BaseReplacer):
|
|||
for v in Utils.get_string_values_from_array(value_node)
|
||||
]
|
||||
|
||||
def traverse(node: Node):
|
||||
def traverse(node: "Node"):
|
||||
execute(node)
|
||||
for child in node.children:
|
||||
traverse(child)
|
||||
|
|
@ -134,7 +134,6 @@ class SourcesReplacer(BaseReplacer):
|
|||
|
||||
for i, file in enumerate(files):
|
||||
file_name = file.get_text_value()
|
||||
print(file_name)
|
||||
if "://" in file_name:
|
||||
source_files.append(file)
|
||||
checksums_str.append(checksums[arch][i])
|
||||
|
|
|
|||
|
|
@ -8,96 +8,93 @@ import tree_sitter_bash as tsbash
|
|||
from tree_sitter import Language, Parser
|
||||
|
||||
from aides_spec.replacers.arch_replacer import ArchReplacer
|
||||
|
||||
# from aides_spec.replacers.checksums_replacer import ChecksumsReplacer
|
||||
# from aides_spec.replacers.local_sources_replacer import LocalSourcesReplacer
|
||||
from aides_spec.replacers.simple_replacer import SimpleReplacer
|
||||
from aides_spec.replacers.sources import SourcesReplacer
|
||||
|
||||
parser_ts = None
|
||||
HEADER = """#
|
||||
# WARNING: Automatically converted from PKGBUILD and may contain errors
|
||||
#
|
||||
"""
|
||||
|
||||
|
||||
def download_pkgbuild_and_all_files(pkgname):
|
||||
class PkgbuildDownloader:
|
||||
"""Handles downloading PKGBUILD and associated files."""
|
||||
|
||||
@staticmethod
|
||||
def download_and_extract(pkgname: str) -> bytes:
|
||||
aur_url = f"https://aur.archlinux.org/{pkgname}.git"
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdirname:
|
||||
try:
|
||||
print(f"Клонируем репозиторий для {pkgname}")
|
||||
print(f"Cloning repository for {pkgname}...")
|
||||
git.Repo.clone_from(aur_url, tmpdirname)
|
||||
print(f"Files for {pkgname} downloaded to {tmpdirname}")
|
||||
|
||||
print(f"Файлы для {pkgname} загружены в {tmpdirname}")
|
||||
|
||||
for root, dirs, files in os.walk(tmpdirname):
|
||||
dirs[:] = [d for d in dirs if d not in [".git"]]
|
||||
files = [
|
||||
file
|
||||
for file in files
|
||||
if file not in ["PKGBUILD", ".SRCINFO"]
|
||||
]
|
||||
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
|
||||
relative_path = os.path.relpath(file_path, tmpdirname)
|
||||
destination_path = os.path.join(os.getcwd(), relative_path)
|
||||
|
||||
os.makedirs(
|
||||
os.path.dirname(destination_path), exist_ok=True
|
||||
)
|
||||
|
||||
shutil.copy(file_path, destination_path)
|
||||
PkgbuildDownloader._copy_files(tmpdirname)
|
||||
|
||||
with open(os.path.join(tmpdirname, "PKGBUILD"), "rb") as f:
|
||||
return f.read()
|
||||
|
||||
return
|
||||
except Exception as e:
|
||||
print(f"Ошибка при скачивании репозитория: {e}")
|
||||
print(f"Error downloading repository: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
@staticmethod
|
||||
def _copy_files(tmpdirname: str):
|
||||
"""Copies all files to the current directory."""
|
||||
for root, dirs, files in os.walk(tmpdirname):
|
||||
dirs[:] = [d for d in dirs if d not in [".git"]]
|
||||
files = [f for f in files if f not in ["PKGBUILD", ".SRCINFO"]]
|
||||
|
||||
def download_pkgbuild(pkgname):
|
||||
return download_pkgbuild_and_all_files(pkgname)
|
||||
for file in files:
|
||||
file_path = os.path.join(root, file)
|
||||
relative_path = os.path.relpath(file_path, tmpdirname)
|
||||
destination_path = os.path.join(os.getcwd(), relative_path)
|
||||
|
||||
os.makedirs(os.path.dirname(destination_path), exist_ok=True)
|
||||
shutil.copy(file_path, destination_path)
|
||||
|
||||
|
||||
def process_file(content, tree, replacers):
|
||||
class PkgbuildProcessor:
|
||||
"""Processes PKGBUILD files with replacers."""
|
||||
|
||||
def __init__(self):
|
||||
self.parser = self._initialize_parser()
|
||||
|
||||
@staticmethod
|
||||
def _initialize_parser() -> Parser:
|
||||
bash_language = Language(tsbash.language())
|
||||
return Parser(bash_language)
|
||||
|
||||
def process(self, content: bytes, replacers: list) -> bytes:
|
||||
tree = self.parser.parse(content)
|
||||
|
||||
for replacer_class in replacers:
|
||||
replacer = replacer_class(content, tree)
|
||||
content = replacer.process()
|
||||
tree = parser_ts.parse(content, tree)
|
||||
tree = self.parser.parse(content, tree)
|
||||
|
||||
return content
|
||||
|
||||
|
||||
HEADER = """#
|
||||
# WARNING: Automatic converted from PKGBUILD and may contains errors
|
||||
#
|
||||
|
||||
"""
|
||||
|
||||
|
||||
def create_from_pkgbuild(content, output_file):
|
||||
global parser_ts
|
||||
BASH_LANGUAGE = Language(tsbash.language())
|
||||
parser_ts = Parser(BASH_LANGUAGE)
|
||||
|
||||
tree = parser_ts.parse(content)
|
||||
def create_from_pkgbuild(content: bytes, output_file: str):
|
||||
"""Creates a new spec file from a PKGBUILD."""
|
||||
processor = PkgbuildProcessor()
|
||||
|
||||
replacers = [
|
||||
SimpleReplacer,
|
||||
ArchReplacer,
|
||||
SourcesReplacer,
|
||||
# LocalSourcesReplacer,
|
||||
# ChecksumsReplacer,
|
||||
]
|
||||
|
||||
new_content = process_file(content, tree, replacers)
|
||||
|
||||
try:
|
||||
new_content = processor.process(content, replacers)
|
||||
new_content = bytes(HEADER, encoding="utf-8") + new_content
|
||||
|
||||
try:
|
||||
with open(output_file, "wb") as f:
|
||||
f.write(new_content)
|
||||
print(f"Файл успешно записан в {output_file}.")
|
||||
|
||||
print(f"File successfully written to {output_file}.")
|
||||
except IOError as e:
|
||||
print(f"Ошибка при записи файла: {e}")
|
||||
print(f"Error writing file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
|
|
|||
Loading…
Reference in a new issue