Moving assets over to Git LFS (#885)

* Moving assets over to Git LFS

* Fixed some typos in the readme

* One more readme readability improvement

.appveyor.yml

@@ -4,6 +4,7 @@ environment:
   BOOST_ROOT: C:/Libraries/boost_1_69_0
   PATH: C:\Ruby25-x64\bin;$(PATH);C:\cygwin64\bin
   VULKAN_SDK: C:/VulkanSDK/1.1.114.0
+  GIT_LFS_SKIP_SMUDGE: 1
 
 image:
   - Visual Studio 2017
@@ -14,14 +15,16 @@ cache:
   - ThirdParty
   - assets
   - VulkanSDK.exe
+  - .git/lfs/objects  
 
 install:
   - gem install os colorize rubyzip json sha3
   - if not exist VulkanSDK.exe curl -L --silent --show-error --output VulkanSDK.exe https://vulkan.lunarg.com/sdk/download/1.1.114.0/windows/VulkanSDK-1.1.114.0-Installer.exe?Human=true
   - VulkanSDK.exe /S
 
 build_script:
-  - ruby SetupThrive.rb --precompiled thrive:thrive
+  - git lfs pull  
+  - ruby SetupThrive.rb --precompiled
 
 test_script:
   - cd build\bin

.circleci/config.yml

@@ -3,29 +3,31 @@ jobs:
    build:
      working_directory: ~/thrive
      docker:
-       - image: hhyyrylainen/leviathan-deps:v5
+       - image: hhyyrylainen/leviathan-deps:v6
+     environment:
+       GIT_LFS_SKIP_SMUDGE: 1
      steps:
        - checkout
        - restore_cache:
            keys:
-             - v4-deps-{{ arch }}-{{ checksum "SetupThrive.rb" }}
-             - v4-deps-{{ arch }}
+             - v1-lfs-{{ .Branch }}
+             - v1-lfs-master
+       - shell:
+           name: Pull Git LFS files, retry until done
+           command: |
+             while true; do git reset --hard HEAD && git clean -fd && git lfs pull && break; done
+       - save_cache:
+          key: v1-lfs-{{ .Branch }}
+          paths:
+            - .git/lfs
        - restore_cache:
            keys:
-             - v3-assets-{{ .Branch }}
-             - v3-assets
+             - v4-deps-{{ arch }}-{{ checksum "SetupThrive.rb" }}
+             - v4-deps-{{ arch }}
        - restore_cache:
            keys:
              - v4-build-{{ arch }}-{{ .Branch }}
        - run: ./SetupThrive.rb --no-packagemanager --no-updates -j 2 --fully-parallel-project-compile --project-parallel 5
-       - save_cache:
-           key: v3-assets-{{ .Branch }}
-           paths:
-             - assets
-       - save_cache:
-           key: v3-assets
-           paths:
-             - assets
        - save_cache:
            key: v4-deps-{{ arch }}-{{ checksum "SetupThrive.rb" }}
            paths:

.gitattributes

@@ -1,5 +1,12 @@
 * text=auto
-
 *.lua text
 *.cpp text
-*.h text
+*.h text
+*.asset filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.mkv filter=lfs diff=lfs merge=lfs -text
+*.ogg filter=lfs diff=lfs merge=lfs -text
+*.ico filter=lfs diff=lfs merge=lfs -text
+*.ttf filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,7 +1,4 @@
-# Asset directory
-assets/*
-
-# autogenerated file
+# autogenerated files
 /src/thrive_version.h
 /src/main.cpp
 /src/generated
@@ -123,5 +120,7 @@ nbproject/
 # Cscope files
 /cscope.files
 /RunCodeIndexing.rb
+
+# JS lint installed files
 /node_modules
 /package.json

.lfsconfig

@@ -0,0 +1,2 @@
+[lfs]
+	url = https://dev.revolutionarygamesstudio.com/api/v1/lfs/thrive

.rubocop.yml

@@ -0,0 +1,6 @@
+Metrics/LineLength:
+  Max: 95
+
+Style/GlobalVars:
+  Enabled: false
+

CMakeLists.txt

@@ -15,23 +15,10 @@ project(Thrive)
 # Cache Variables #
 ###################
 
-set(ASSET_DIRECTORY "${CMAKE_SOURCE_DIR}/assets"
-  CACHE PATH "Path to assets"
-  )
-
 # This isn't really meant to be modified
 set(LEVIATHAN_SRC "${PROJECT_SOURCE_DIR}/ThirdParty/Leviathan" CACHE FILEPATH
   "Path to leviathan source folder")
 
-if(NOT IS_DIRECTORY "${ASSET_DIRECTORY}/raw_assets")
-    message(FATAL_ERROR 
-"Could not find assets in ${ASSET_DIRECTORY}.  
-Please run the SetupThrive.rb script to download them.
-Use 'thrive' for the username and password
-"
-    )
-endif()
-
 
 ###########
 # Options #

ImportAssets.rb

@@ -8,45 +8,63 @@
 
 require_relative 'RubySetupSystem/RubyCommon'
 
-editor = "ThirdParty/Leviathan/build/bin/LeviathanEditor"
+ASSETS_REPO = 'https://github.com/Revolutionary-Games/Thrive-Raw-Assets.git'.freeze
+ASSETS_FOLDER = '../Thrive-Raw-Assets'.freeze
 
-if OS.windows?
-  editor += ".exe"
+unless File.exist? ASSETS_FOLDER
+  info "Raw assets folder (#{ASSETS_FOLDER}) doesn't exist. Trying to clone it."
+  puts 'If you are using ssh to access Github you should manually clone it instead...'
+
+  runOpen3Checked('git', 'clone', ASSETS_REPO, ASSETS_FOLDER)
+
+  onError 'Failed to clone assets repo' unless File.exist? ASSETS_FOLDER
 end
 
-if !File.exists? editor
+puts 'Note: this script does not automatically pull the raw assets repo. ' \
+     'Make sure you are using the right commit manually.'
+
+info "Doing git lfs pull to make sure it's up to date"
 
-  onError "LeviathanEditor is not at expected path"
+Dir.chdir(ASSETS_FOLDER) do
+  runOpen3Checked('git', 'lfs', 'pull')
 end
 
+success 'Done making sure raw assets repo is good'
+
+editor = 'ThirdParty/Leviathan/build/bin/LeviathanEditor'
+
+editor += '.exe' if OS.windows?
+
+onError 'LeviathanEditor is not at expected path' unless File.exist? editor
+
 editor = File.realpath editor
 
-info "Importing assets. When importing shaders, this may take multiple minutes."
+info 'Importing assets. When importing shaders, this may take multiple minutes.'
 
-Open3.popen3(editor, "--import", File.realpath("shaders"), File.realpath("assets"),
-             "--import", File.realpath("assets/raw_assets"), File.realpath("assets"),
-             chdir: File.realpath(File.dirname editor)) {
-  |stdin, stdout, stderr, wait_thr|
+Open3.popen3(editor, '--import', File.realpath('shaders'), File.realpath('assets'),
+             '--import', File.realpath(File.join(ASSETS_FOLDER, 'raw_assets')),
+             File.realpath('assets'),
+             chdir: File.realpath(File.dirname(editor))) do |_stdin, stdout, stderr, wait_thr|
 
-  outThread = Thread.new{
-    stdout.each {|line|
+  out_thread = Thread.new do
+    stdout.each do |line|
       puts line
-    }
-  }
+    end
+  end
 
-  errThread = Thread.new{
-    stderr.each {|line|
-      puts (line).red
-    }
-  }
+  err_thread = Thread.new do
+    stderr.each do |line|
+      puts line.red
+    end
+  end
 
   exit_status = wait_thr.value
-  outThread.join
-  errThread.join
+  out_thread.join
+  err_thread.join
 
-  if exit_status != 0
-    onError "Failed to run import with LeviathanEditor"
-  end
-}
+  onError 'Failed to run import with LeviathanEditor' if exit_status != 0
+end
 
-success "Done importing"
+info "NOTE: import failures aren't reported as exit codes so this script " \
+     "can't currently detect failures (other than serious ones)"
+success 'Done importing'

README.md

@@ -2,12 +2,12 @@ Thrive
 ======
 
 This is the code repository for Thrive. For more information, visit
-[Revolutionary Games' Website](http://revolutionarygamesstudio.com/).
+[Revolutionary Games' Website](https://revolutionarygamesstudio.com/).
 
 Overview
 ----------------
 Repository structure:
-- assets: An SVN repository that doesn't follow with the git repository. This contains all the assets such as models and other binaries. It is located at [asset repository][asset_repository] but it will be automatically downloaded by the setup script.
+- assets: This folder contains all the assets such as models and other binaries. The big files in this folder use [Git LFS](https://git-lfs.github.com/) in order to keep this repository from bloating. You need to have Git LFS installed to get the files.
 - doc: Documentation files. Contains style guide, engine overview and other useful documentation.
 - scripts: AngelScript scripts that contain part of the codebase. Scripts are used for easier development and code here can then later be transferred to the C++ base for performance. 
 - src: The C++ code base containing the common helper classes and gameplay code moved to C++.
@@ -22,7 +22,7 @@ There are also other useful documents in the doc folder not mentioned here.
 
 ### Script Authors
 If you only want to modify the AngelScript scripts, you can obtain a 
-working copy of the game from official releases here: http://revolutionarygamesstudio.com/releases/
+working copy of the game from official releases [here][releasespage],
 alternatively you can request a newer version from developers or compile the project yourself.
 
 Be sure to have a look at the [styleguide][styleguide],
@@ -37,31 +37,36 @@ both for guidelines on code formatting and git usage.
 
 ### Modellers, texture and GUI artists, and Sound Engineers
 To work on the art assets you will want a working copy of the game.
-You can find official releases here: http://revolutionarygamesstudio.com/releases/
+You can find official releases [here][releasespage],
 alternatively you can request a newer version from developers, or compile the project yourself.
- 
+
 After you have obtained a working version of the game, you can place any new assets in the corresponding subdirectories:
-bin/Data/Sound, bin/Data/Models, bin/Data/Materials and gui and the game will automatically detect up your new files, which you can then use in scripts.
+bin/Data/Sound, bin/Data/Models, or bin/Data/Textures. The game will automatically detect your new files,
+which you can then use in scripts.
 An example of modifying a script to use your model would be to open scripts/microbe_stage/organelle_table.as with a text editor and 
 find 'nucleusParameters.mesh = "nucleus.fbx";' that sets the model used by the nucleus and change that to your new model file.
 Similarly you can find sections of the scripts that use other assets and replace the assets they use.
+If you are replacing an imported asset file (file extension `.asset`) you need to use the [import tool][importtutorial] to convert 
+it before the game can use the file. Also if you are adding a new asset that needs to be in converted form (models, textures, 
+shaders, etc.) you need to convert the file before using.
+
 If you are truly uncomfortable with editing scripts you can simply try stealing the names of existing assets. For example 
 going into the sound subdirectory and stealing the name "microbe-theme-1.ogg" by renaming your new sound-file to that and the 
 game will then play that sound instead.
 
-To contribute assets you can contact a developer and provide that person with your assets and the developer can add the assets 
-the official repository. It will at a later time be possible to commit to the subversion (SVN) server yourself. In the meantime you can
-learn about subversion from a lot of easy-to-find tutorials. A recommended tool for windows is: [tortoise svn][tortoiseSVN].
+To contribute assets you can contact a developer and provide that person with your assets and the developer can add the assets to
+the official repository. It will at a later time be possible to [commit](https://wiki.revolutionarygamesstudio.com/wiki/Git_LFS) to
+Git LFS server yourself, currently it is limited to only Thrive developers. Note that you must have Git LFS installed.
 
 Extra note for modellers:
-Ogre (the graphics system that Thrive uses) uses it's own file format for materials, models (meshes) and skeletons. In order for your 
-model to be used in Thrive it will need to be converted. You can contact a developer to do this for you or you can do it yourself.
-A good tutorial for converting blender files can be found [here][blender_ogre_tutorial] Note that you should make sure to use a version of
-blender that has a corresponding version of blender2ogre to do the conversion.
+You should export your model in FBX format from blender or other
+modelling software you use, and then use the [import
+tool][importtutorial] to convert the model to an usable form. If you
+are importing a model with a skeleton or morph animations you'll need
+a special import configuration file.
 
-[blender_ogre_tutorial]: http://thrivegame.wikidot.com/blender-and-ogre-tutorial "Blender to ogre tutorial"
-[asset_repository]: https://boostslair.com/svn/thrive_assets "Asset Repository"
-[tortoiseSVN]: http://tortoisesvn.net/docs/release/TortoiseSVN_en/ "Tortoise SVN"
+[releasespage]: https://revolutionarygamesstudio.com/releases/
 [styleguide]: doc/style_guide.md "Styleguide"
 [setupguide]: doc/setup_instructions.md
 [asprimer]: doc/angelscript_primer.md "AngelScript primer"
+[importtutorial]: https://wiki.revolutionarygamesstudio.com/wiki/How_to_Import_Assets "How to import assets"