Add python script to map files to directory structure, use compression for package

.gitlab-ci.yml

@@ -67,9 +67,12 @@ test_and_build_frontend:
 .package_server:
     image: alpine:latest
     stage: package
+    before_script:
+        - apk update
+        - apk add xz tar
     artifacts:
         paths:
-            - '*.tar'
+            - '*.tar.xz'
 
 package_server_glibc:
     extends: .package_server
@@ -81,7 +84,7 @@ package_server_glibc:
     script:
         - mkdir static
         - mv frontend/dist/* static/
-        - tar -cvf linux-x64-glibc.tar server static
+        - tar -cvJf linux-x64-glibc.tar.xz server static
 
 package_server_muslc:
     extends: .package_server
@@ -93,7 +96,7 @@ package_server_muslc:
     script:
         - mkdir static
         - mv frontend/dist/* static/
-        - tar -cvf linux-x64-muslc.tar server static
+        - tar -cvJf linux-x64-muslc.tar.xz server static
 
 upload_assets:
     stage: release
@@ -106,8 +109,8 @@ upload_assets:
     rules:
         - if: $CI_COMMIT_TAG
     script:
-        - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-glibc.tar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar"'
+        - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-glibc.tar.xz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar.xz"'
-        - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-muslc.tar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar"'
+        - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-muslc.tar.xz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar.xz"'
 
 create_release:
     stage: release
@@ -123,9 +126,9 @@ create_release:
         description: 'Release $CI_COMMIT_TAG'
         assets:
             links:
-                - name: 'linux-x64-glibc.tar'
+                - name: 'linux-x64-glibc.tar.xz'
-                  url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar'
+                  url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar.xz'
                   link_type: package
-                - name: 'linux-x64-muslc.tar'
+                - name: 'linux-x64-muslc.tar.xz'
-                  url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar'
+                  url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar.xz'
                   link_type: package

fs_dump/.gitignore

@@ -0,0 +1,121 @@
+# Created by https://www.toptal.com/developers/gitignore/api/pycharm
+# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm
+
+### PyCharm ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# SonarLint plugin
+.idea/sonarlint/
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm Patch ###
+# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
+
+# *.iml
+# modules.xml
+# .idea/misc.xml
+# *.ipr
+
+# Sonarlint plugin
+# https://plugins.jetbrains.com/plugin/7973-sonarlint
+.idea/**/sonarlint/
+
+# SonarQube Plugin
+# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
+.idea/**/sonarIssues.xml
+
+# Markdown Navigator plugin
+# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
+.idea/**/markdown-navigator.xml
+.idea/**/markdown-navigator-enh.xml
+.idea/**/markdown-navigator/
+
+# Cache file creation bug
+# See https://youtrack.jetbrains.com/issue/JBR-2257
+.idea/$CACHE_FILE$
+
+# CodeStream plugin
+# https://plugins.jetbrains.com/plugin/12206-codestream
+.idea/codestream.xml
+
+# Azure Toolkit for IntelliJ plugin
+# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
+.idea/**/azureSettings.xml
+
+# End of https://www.toptal.com/developers/gitignore/api/pycharm
+
+dump
+files
+sqlite.db

fs_dump/.idea/fs_dump.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.10" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

fs_dump/.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

fs_dump/.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10" project-jdk-type="Python SDK" />
+</project>

fs_dump/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/fs_dump.iml" filepath="$PROJECT_DIR$/.idea/fs_dump.iml" />
+    </modules>
+  </component>
+</project>

fs_dump/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$/.." vcs="Git" />
+  </component>
+</project>

fs_dump/create_disks.py

@@ -0,0 +1,66 @@
+from peewee import SqliteDatabase, Model, AutoField, IntegerField, TextField, BlobField
+from pathlib import Path
+import shutil
+
+db = SqliteDatabase('sqlite.db')
+dump_path = Path('dump')
+
+class User(Model):
+    class Meta:
+        table_name = 'user'
+        database = db
+
+    id = AutoField()
+    gitlab = IntegerField()
+    name = TextField()
+    password = TextField()
+    role = IntegerField()
+    root_id = IntegerField()
+    tfa_type = IntegerField()
+    tfa_secret = BlobField()
+    gitlab_at = TextField()
+    gitlab_rt = TextField()
+
+
+class INode(Model):
+    class Meta:
+        table_name = 'inode'
+        database = db
+
+    id = AutoField()
+    is_file = IntegerField()
+    name = TextField()
+    parent_id = IntegerField()
+    owner_id = IntegerField()
+    size = IntegerField()
+    has_preview = IntegerField()
+
+
+def parse_inode(node: INode, path: Path):
+    new_path: Path = path / node.name
+    if node.is_file == 0:
+        new_path.mkdir()
+        children = INode.select().where(INode.parent_id == node.id)
+        for child in children:
+            parse_inode(child, new_path)
+    else:
+        shutil.copy(f'files/{node.id}', new_path)
+
+
+def parse_user(user: User):
+    path: Path = dump_path / user.name
+    path.mkdir()
+    root_children = INode.select().where(INode.parent_id == user.root_id)
+    for child in root_children:
+        parse_inode(child, path)
+
+
+if __name__ == '__main__':
+    if dump_path.exists():
+        shutil.rmtree(dump_path)
+    dump_path.mkdir()
+
+    db.connect()
+    users = User.select()
+    for user in users:
+        parse_user(user)