Add python script to map files to directory structure, use compression for package

2022-09-04 14:41:21 +02:00 · 2022-09-04 14:41:21 +02:00 · 312de38f1c
commit 312de38f1c
parent 033a0aca2e
8 changed files with 231 additions and 9 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -67,9 +67,12 @@ test_and_build_frontend:
 .package_server:
    image: alpine:latest
    stage: package
+    before_script:
+        - apk update
+        - apk add xz tar
    artifacts:
        paths:
-            - '*.tar'
+            - '*.tar.xz'

 package_server_glibc:
    extends: .package_server
@ -81,7 +84,7 @@ package_server_glibc:
    script:
        - mkdir static
        - mv frontend/dist/* static/
-        - tar -cvf linux-x64-glibc.tar server static
+        - tar -cvJf linux-x64-glibc.tar.xz server static

 package_server_muslc:
    extends: .package_server
@ -93,7 +96,7 @@ package_server_muslc:
    script:
        - mkdir static
        - mv frontend/dist/* static/
-        - tar -cvf linux-x64-muslc.tar server static
+        - tar -cvJf linux-x64-muslc.tar.xz server static

 upload_assets:
    stage: release
@ -106,8 +109,8 @@ upload_assets:
    rules:
        - if: $CI_COMMIT_TAG
    script:
-        - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-glibc.tar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar"'
-        - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-muslc.tar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar"'
+        - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-glibc.tar.xz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar.xz"'
+        - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-muslc.tar.xz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar.xz"'

 create_release:
    stage: release
@ -123,9 +126,9 @@ create_release:
        description: 'Release $CI_COMMIT_TAG'
        assets:
            links:
-                - name: 'linux-x64-glibc.tar'
-                  url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar'
+                - name: 'linux-x64-glibc.tar.xz'
+                  url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar.xz'
                  link_type: package
-                - name: 'linux-x64-muslc.tar'
-                  url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar'
+                - name: 'linux-x64-muslc.tar.xz'
+                  url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar.xz'
                  link_type: package
--- a/fs_dump/.gitignore
+++ b/fs_dump/.gitignore
@ -0,0 +1,121 @@
+# Created by https://www.toptal.com/developers/gitignore/api/pycharm
+# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+# Azure Toolkit for IntelliJ plugin
+# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
+.idea/**/azureSettings.xml
+
+# End of https://www.toptal.com/developers/gitignore/api/pycharm
+
+dump
+files
+sqlite.db
--- a/fs_dump/.idea/fs_dump.iml
+++ b/fs_dump/.idea/fs_dump.iml
@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.10" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>
--- a/fs_dump/.idea/inspectionProfiles/profiles_settings.xml
+++ b/fs_dump/.idea/inspectionProfiles/profiles_settings.xml
@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>
--- a/fs_dump/.idea/misc.xml
+++ b/fs_dump/.idea/misc.xml
@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10" project-jdk-type="Python SDK" />
+</project>
--- a/fs_dump/.idea/modules.xml
+++ b/fs_dump/.idea/modules.xml
@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/fs_dump.iml" filepath="$PROJECT_DIR$/.idea/fs_dump.iml" />
+    </modules>
+  </component>
+</project>
--- a/fs_dump/.idea/vcs.xml
+++ b/fs_dump/.idea/vcs.xml
@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$/.." vcs="Git" />
+  </component>
+</project>
--- a/fs_dump/create_disks.py
+++ b/fs_dump/create_disks.py
@ -0,0 +1,66 @@
+from peewee import SqliteDatabase, Model, AutoField, IntegerField, TextField, BlobField
+from pathlib import Path
+import shutil
+
+db = SqliteDatabase('sqlite.db')
+dump_path = Path('dump')
+
+class User(Model):
+    class Meta:
+        table_name = 'user'
+        database = db
+
+    id = AutoField()
+    gitlab = IntegerField()
+    name = TextField()
+    password = TextField()
+    role = IntegerField()
+    root_id = IntegerField()
+    tfa_type = IntegerField()
+    tfa_secret = BlobField()
+    gitlab_at = TextField()
+    gitlab_rt = TextField()
+
+
+class INode(Model):
+    class Meta:
+        table_name = 'inode'
+        database = db
+
+    id = AutoField()
+    is_file = IntegerField()
+    name = TextField()
+    parent_id = IntegerField()
+    owner_id = IntegerField()
+    size = IntegerField()
+    has_preview = IntegerField()
+
+
+def parse_inode(node: INode, path: Path):
+    new_path: Path = path / node.name
+    if node.is_file == 0:
+        new_path.mkdir()
+        children = INode.select().where(INode.parent_id == node.id)
+        for child in children:
+            parse_inode(child, new_path)
+    else:
+        shutil.copy(f'files/{node.id}', new_path)
+
+
+def parse_user(user: User):
+    path: Path = dump_path / user.name
+    path.mkdir()
+    root_children = INode.select().where(INode.parent_id == user.root_id)
+    for child in root_children:
+        parse_inode(child, path)
+
+
+if __name__ == '__main__':
+    if dump_path.exists():
+        shutil.rmtree(dump_path)
+    dump_path.mkdir()
+
+    db.connect()
+    users = User.select()
+    for user in users:
+        parse_user(user)