From 312de38f1c796a3cb2e0d3f57c5f8d5015fead74 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 4 Sep 2022 14:41:21 +0200 Subject: [PATCH] Add python script to map files to directory structure, use compression for package --- .gitlab-ci.yml | 21 +-- fs_dump/.gitignore | 121 ++++++++++++++++++ fs_dump/.idea/fs_dump.iml | 8 ++ .../inspectionProfiles/profiles_settings.xml | 6 + fs_dump/.idea/misc.xml | 4 + fs_dump/.idea/modules.xml | 8 ++ fs_dump/.idea/vcs.xml | 6 + fs_dump/create_disks.py | 66 ++++++++++ 8 files changed, 231 insertions(+), 9 deletions(-) create mode 100644 fs_dump/.gitignore create mode 100644 fs_dump/.idea/fs_dump.iml create mode 100644 fs_dump/.idea/inspectionProfiles/profiles_settings.xml create mode 100644 fs_dump/.idea/misc.xml create mode 100644 fs_dump/.idea/modules.xml create mode 100644 fs_dump/.idea/vcs.xml create mode 100644 fs_dump/create_disks.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8cae5a7..2cad9a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -67,9 +67,12 @@ test_and_build_frontend: .package_server: image: alpine:latest stage: package + before_script: + - apk update + - apk add xz tar artifacts: paths: - - '*.tar' + - '*.tar.xz' package_server_glibc: extends: .package_server @@ -81,7 +84,7 @@ package_server_glibc: script: - mkdir static - mv frontend/dist/* static/ - - tar -cvf linux-x64-glibc.tar server static + - tar -cvJf linux-x64-glibc.tar.xz server static package_server_muslc: extends: .package_server @@ -93,7 +96,7 @@ package_server_muslc: script: - mkdir static - mv frontend/dist/* static/ - - tar -cvf linux-x64-muslc.tar server static + - tar -cvJf linux-x64-muslc.tar.xz server static upload_assets: stage: release @@ -106,8 +109,8 @@ upload_assets: rules: - if: $CI_COMMIT_TAG script: - - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-glibc.tar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar"' - - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-muslc.tar "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar"' + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-glibc.tar.xz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar.xz"' + - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file linux-x64-muslc.tar.xz "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar.xz"' create_release: stage: release @@ -123,9 +126,9 @@ create_release: description: 'Release $CI_COMMIT_TAG' assets: links: - - name: 'linux-x64-glibc.tar' - url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar' + - name: 'linux-x64-glibc.tar.xz' + url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-glibc/$CI_COMMIT_TAG/linux-x64-glibc.tar.xz' link_type: package - - name: 'linux-x64-muslc.tar' - url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar' + - name: 'linux-x64-muslc.tar.xz' + url: '${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/linux-x64-muslc/$CI_COMMIT_TAG/linux-x64-muslc.tar.xz' link_type: package diff --git a/fs_dump/.gitignore b/fs_dump/.gitignore new file mode 100644 index 0000000..7218095 --- /dev/null +++ b/fs_dump/.gitignore @@ -0,0 +1,121 @@ +# Created by https://www.toptal.com/developers/gitignore/api/pycharm +# Edit at https://www.toptal.com/developers/gitignore?templates=pycharm + +### PyCharm ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# SonarLint plugin +.idea/sonarlint/ + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### PyCharm Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +# Azure Toolkit for IntelliJ plugin +# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij +.idea/**/azureSettings.xml + +# End of https://www.toptal.com/developers/gitignore/api/pycharm + +dump +files +sqlite.db \ No newline at end of file diff --git a/fs_dump/.idea/fs_dump.iml b/fs_dump/.idea/fs_dump.iml new file mode 100644 index 0000000..8437fe6 --- /dev/null +++ b/fs_dump/.idea/fs_dump.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/fs_dump/.idea/inspectionProfiles/profiles_settings.xml b/fs_dump/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/fs_dump/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/fs_dump/.idea/misc.xml b/fs_dump/.idea/misc.xml new file mode 100644 index 0000000..dc9ea49 --- /dev/null +++ b/fs_dump/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/fs_dump/.idea/modules.xml b/fs_dump/.idea/modules.xml new file mode 100644 index 0000000..cfd8988 --- /dev/null +++ b/fs_dump/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/fs_dump/.idea/vcs.xml b/fs_dump/.idea/vcs.xml new file mode 100644 index 0000000..6c0b863 --- /dev/null +++ b/fs_dump/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/fs_dump/create_disks.py b/fs_dump/create_disks.py new file mode 100644 index 0000000..25c0d03 --- /dev/null +++ b/fs_dump/create_disks.py @@ -0,0 +1,66 @@ +from peewee import SqliteDatabase, Model, AutoField, IntegerField, TextField, BlobField +from pathlib import Path +import shutil + +db = SqliteDatabase('sqlite.db') +dump_path = Path('dump') + +class User(Model): + class Meta: + table_name = 'user' + database = db + + id = AutoField() + gitlab = IntegerField() + name = TextField() + password = TextField() + role = IntegerField() + root_id = IntegerField() + tfa_type = IntegerField() + tfa_secret = BlobField() + gitlab_at = TextField() + gitlab_rt = TextField() + + +class INode(Model): + class Meta: + table_name = 'inode' + database = db + + id = AutoField() + is_file = IntegerField() + name = TextField() + parent_id = IntegerField() + owner_id = IntegerField() + size = IntegerField() + has_preview = IntegerField() + + +def parse_inode(node: INode, path: Path): + new_path: Path = path / node.name + if node.is_file == 0: + new_path.mkdir() + children = INode.select().where(INode.parent_id == node.id) + for child in children: + parse_inode(child, new_path) + else: + shutil.copy(f'files/{node.id}', new_path) + + +def parse_user(user: User): + path: Path = dump_path / user.name + path.mkdir() + root_children = INode.select().where(INode.parent_id == user.root_id) + for child in root_children: + parse_inode(child, path) + + +if __name__ == '__main__': + if dump_path.exists(): + shutil.rmtree(dump_path) + dump_path.mkdir() + + db.connect() + users = User.select() + for user in users: + parse_user(user)