diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fcadb2c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text eol=lf diff --git a/.github/pack_launcher.sh b/.github/pack_launcher.sh new file mode 100755 index 0000000..3b64342 --- /dev/null +++ b/.github/pack_launcher.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Copyright (c) 2021 José Manuel Barroso Galindo + +set -euo pipefail + +if ! gh release list | grep -q "latest" ; then + gh release create "latest" || true + sleep 15s +fi + +zip "MiSTer_Downloader.zip" downloader.sh +gh release upload "latest" "MiSTer_Downloader.zip" --clobber diff --git a/.github/release.sh b/.github/release.sh new file mode 100755 index 0000000..75740e4 --- /dev/null +++ b/.github/release.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Copyright (c) 2021 José Manuel Barroso Galindo + +set -euo pipefail + +git add dont_download.sh +git commit -m "BOT: New dont_download.sh" > /dev/null 2>&1 || true +git fetch origin main + +set +e +CHANGES="$(git diff main:dont_download.sh origin/main:dont_download.sh | sed '/^[+-]export COMMIT/d' | sed '/^+++/d' | sed '/^---/d' | grep '^[+-]' | wc -l)" +set -e + +if [ ${CHANGES} -ge 1 ] ; then + echo "There are changes to push." + echo + git push origin main + echo + echo "New dont_download.sh can be used." + echo "::set-output name=NEW_RELEASE::yes" +else + echo "Nothing to be updated." + echo "::set-output name=NEW_RELEASE::no" +fi diff --git a/.github/workflows/all_tests.yml b/.github/workflows/all_tests.yml new file mode 100644 index 0000000..1ee8c4a --- /dev/null +++ b/.github/workflows/all_tests.yml @@ -0,0 +1,31 @@ +name: All Tests + +on: + pull_request: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-20.04 + + steps: + - name: Install sharutils + run: sudo apt-get install sharutils + + - uses: actions/setup-python@v2 + with: + python-version: '3.5' + + - uses: actions/checkout@v2 + + - name: Unit Tests + run: cd src && python3 -m unittest discover -s test/unit + + - name: Integration Tests + run: cd src && python3 -m unittest discover -s test/integration + + - name: System Quick Tests + run: cd src && python3 -m unittest discover -s test/system/quick + + - name: System Slow Tests + run: cd src && python3 -m unittest discover -s test/system/slow diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..5cd5275 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,40 @@ +name: Build + +on: push + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - name: Install sharutils + run: sudo apt-get install sharutils + + - uses: actions/setup-python@v2 + with: + python-version: '3.5' + + - uses: actions/checkout@v2 + + - name: Unit Tests + run: cd src && python3 -m unittest discover -s test/unit + + - name: Integration Tests + run: cd src && python3 -m unittest discover -s test/integration + + - name: System Quick Tests + run: cd src && python3 -m unittest discover -s test/system/quick + + - name: Build + run: ./src/build.sh > dont_download.sh + + - name: Release + id: release + run: | + git config --global user.email "theypsilon@gmail.com" + git config --global user.name "The CI/CD Bot" + ./.github/release.sh + + - name: System Slow Tests + if: steps.release.outputs.NEW_RELEASE == 'yes' + run: cd src && python3 -m unittest discover -s test/system/slow \ No newline at end of file diff --git a/.github/workflows/pack_launcher.yml b/.github/workflows/pack_launcher.yml new file mode 100644 index 0000000..0eaa2e2 --- /dev/null +++ b/.github/workflows/pack_launcher.yml @@ -0,0 +1,20 @@ +name: Pack Launcher + +on: + push: + paths: + - 'downloader.sh' + workflow_dispatch: + +jobs: + pack: + runs-on: ubuntu-20.04 + + steps: + + - uses: actions/checkout@v2 + + - name: Pack Launcher + run: ./.github/pack_launcher.sh + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4e9e2d1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.idea +venv +__pycache__ +mister.ip +dont_download.ini \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..f7e080f --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# MiSTer Downloader + +This tool installs and **updates all the cores** and other extra files for your *MiSTer*. It also updates the menu core, the MiSTer firmware and the Linux system. The source for all downloads is the [MiSTer Distribution](https://github.com/MiSTer-devel/Distribution_MiSTer) repository. + +The **MiSTer Downloader** is a substitute for the [MiSTer Updater](https://github.com/MiSTer-devel/Updater_script_MiSTer), and is meant to offer a more safe and robust experience, while being much faster. + +As a drawback, the **Downloader** is not backwards compatible with the old INI files that were configured for the [MiSTer Updater](https://github.com/MiSTer-devel/Updater_script_MiSTer). In fact, as of today, this tool doesn't implement many fine-grained features that allow you to customize the updating process in depth. In case you value these features, consider to keep using the [MiSTer Updater](https://github.com/MiSTer-devel/Updater_script_MiSTer) as usual. Both tools will coexist in the near future. + +### Setup and Usage + +Download this [ZIP file](https://github.com/MiSTer-devel/Downloader_MiSTer/releases/download/latest/MiSTer_Downloader.zip) and extract `downloader.sh` to your `/Scripts` folder on your primary SD card (create that folder if it doesn't exist). You only need to perform this operation once, since this tool self-updates itself. + +To use it, on your MiSTer main menu, go to the *Scripts* screen, and select `downloader`. + +### Options + +You may create a `downloader.ini` file to tweak some parameters. + +Here you can see the default parameters and the options that you may change: + +```ini +[MiSTer] +; base_path is where most files will be installed +; Useful for setups with USB storage, for example: '/media/usb0/' +base_path = '/media/fat/' + +; base_system_path is where system files such as 'MiSTer' and 'menu.rbf' will be installed. +; Warning: It is recommended to NOT change this setting regardless of your setup. +base_system_path = '/media/fat/' + +; allow_delete options: +; 0 -> Don't allow this tool to delete anything at all. +; 1 -> Allow this tool to delete any old file from previous updates. +; 2 -> Allow this tool to delete only old cores that receive a new version. +allow_delete = 1 + +; allow_reboot options: +; 0 -> Don't allow this tool to ever reboot automatically. +; 1 -> Allow this tool to reboot the system after any system file has been updated. +; 2 -> Allow this tool to reboot the system only after Linux has been updated. +allow_reboot = 1 + +; update_linux options: +; true -> Updates Linux when there is a new update (very recommended). +; false -> Doesn't update Linux. +update_linux = true + +; parallel_update options: +; true -> Tries to more than one file simultaneously. +; false -> Will only download one file at a time. +parallel_update = true + +; downloader_timeout: Can be tweaked to increase the timeout time in seconds +; It is useful to increase this value for users with slow connections. +downloader_timeout = 300 + +; downloader_retries: Can be tweaked to increase the retries per failed download +; It is useful to increase this value for users with very unstable connections. +downloader_retries = 3 +``` + +### Roadmap + +- [x] Initial Release +- [ ] [Cheats](https://gamehacking.org/mister/) fetching +- [ ] First-run optimisations +- [ ] Configurable custom download filters +- [ ] Handle duplicated `games` folders through symlinks (*GBA* <-> *GBA2P*, and *GAMEBOY* <-> *GAMEBOY2P*) +- [ ] Integration with *MiSTer* binary diff --git a/dont_download.sh b/dont_download.sh new file mode 100755 index 0000000..feb49f0 --- /dev/null +++ b/dont_download.sh @@ -0,0 +1,224 @@ +#!/usr/bin/env bash +set -euo pipefail +export DOWNLOADER_LAUNCHER_PATH="${DOWNLOADER_LAUNCHER_PATH:-${0}}" +export COMMIT="f707924" +uudecode -o - "${0}" | xzcat -d -c > "/tmp/dont_download.zip" +chmod a+x "/tmp/dont_download.zip" +"/tmp/dont_download.zip" +exit 0 +begin 644 - +M_3=Z6%H```3FUK1&`@`A`18```!T+^6CX-KX)1E=`!&(0D>*(S/#=P\S_567 +M[:?'\.HJ0S;*+X&N0"(_4GSQN/%D_5&./+C:&X8;_)^7'HI@C@LWLP"IF@-K +M`>]2*R[6OV6YA.E2)Z%$U?)H7<'^K(;U0N\1YT)(G-,L'BJSD7=2UAI!@.D: +M:T3(=^I4!U@NY>)[=^&2)A-HO=;,E$ELOCY(!:84F)PQ&OXC/#WB +M=UY,!'3HM_>D.SOY.=%`G!(,?)P))L[D9CFLX,85.Z$>8V!SI9R;XG,>,,E= +M[GAO!N!V@_=Z?XCSC8F(<_F`(17Q,?MUV9")=!B9@K5X9S1AURI_/3%F`=^Q +MY;*([,8/`R5K(_0:U\%8,OAX1ZZ7'AO:WL`#3CP=*M!3:34R#!]^);W:91]= +MATL+MJEXUA;(QS,6/:7&5"PFR[IY/)Q#P;$[Q0M19Y*BJPF(5[$4K>05#H&_ +M%@P/`]^Q*!/^FH& +M@+*D2I",=AE"5'*=<+R0[W9`V\YLO7> +MVLX'FDJ)'>?3RJ$Z9+EMUR(CG*+9P^$C\T>CT;9I%3!,:">5X:C7 +M5^#<%B_UY%#:>)FE[QH,G6TS5_W7KXWH\O5P>IUG6OUHXTC"27KG_!**7:-U +M/"BE\'P;HA?`E7AT1`A??K#C#-;B.4G>FS(FK!SH>'DMP98,;=O&2$I&(D2Z +M_1KJ<^R=(H<4%(0"G&M,"+6[@ENB:>XS?:/%0'YFK1< +M#ZV>,BPC\\4I.WN>O\]5&M]5QFSS!:R\^\CY%]+?+U4.F:VW#:7 +MMH=C_MK=ED*^1#:<:I-T;MW6.T.H(OU4ZW59N7RF@;P=*53I)L0,S&W$:0`_ +M,/M)\LC&?JGWQC6O@+FTDA0E@5()T9-G/GQX-E_[U75?DK5$-GJ2$.E[Z9YO;!["Q[Q>C6C[:"[%8*PQWU%9%V`I]"WO^\PVD!/;&CY +MV1ISJ?1C[F*]F9&;/E$351\J/_5!=HOJ,OL]#D@UJL5^Z(ORJKF2&%'4X[S> +MEQE[]*33GH<]WF=J:]&\/9'M0;Y=-/;TTUXGF^VU]Z-R,)6E6JTNNH?,Z"FC +M!PJ=:(3GO[79%*<130*N3#U=OM_QV/<`Z/UJ()(E^/8GEX0(KN,9'_S,%"C. +M6+Q)1]S)?D'L%TUP]R!H2%2Q8I(!:6F6(61N&C9!3+O9_&4T$.)TCA]OJK-P/TK6^8N*H=PO3.U6)^5@%:Y/#O/HX&<5_K3C.)'-Q68P0TK@F709PL#D;Y-*@[Q[MD`%@5TP +M?[<7O<['?\+'VE<'[58*_R]4M\*";]-0=RBQM4';P#[Q]=E3D&:@Y5DC%>N- +M?&(K!E$(*SMJ/\5,U%@)RQ)N]OI%Z!=>P`]S+O"K=4+J@$^[PSRQS2[B=:RT +M8:4Y!3D"RLY/RY?PVX9GK4"7H"*-P&"]!6/2B@:_LIT&H/'7#++M9Z5: +M37W9\K)@/XM<-#9.CS5WV0KT8*DY1I$&--\\7V)UQ5EMR&P2]NH1F'7/^ +M4YN^27%TG-)I,:JPOCZC#Y)[UET$EDVSH2Y[QKNZ1Z)ZK +MG"0+^)GD!,^T33=OV[SACF:!'\<'Z-I*HQO7/5F`PQ>\ET5$WI4`G@UF7+WX +MLPAZ-F1\6?9,D:,Z7W*:SQE?URM:4B!),+*<]@/QO1Z99$&-10UV;9K!;X^& +M3QLG",8?4\EVL8I>JI##,J7*-:Y0SSV69;I6H(FR.(!E71?&?+#XQ\O7`2+K6+F^AY;)-I.AK2>!@?(5,3]K_#?Y^ +M[]2`)O1UZ"9I/?N#J8W5"'1%]39L$HPU?(5-I8YRW,P_0Q.D4<2Q*"S0*Z,9 +MFW0NA(4_3!RGO.ZC)0Q[VI>MT&G]Y'';'LM6:(&$*&E3F^'V0Y"&`-(4;HGR +MG@6N=I"9+:8AFO7[43J0+\`S)ZF#G.1&7%O@+C#V:A9(5^;7CK2Y.7G]/3[= +MPB#1^GZK@L<$@P3R_Z06R=9(S@JF=T/3EE5J3Q_H,&$2SI"'WV@ZY"LK[QP] +MH2G^G!\4U)[$`=S#9[D&C18W:CSXZ5UI$CS)*SE]'7'ZN?#W>SC>+OQ6PZO- +MPA\[;9.>V7HO&SH*TN&P[*;V^:'-,8E%+,L\`-PO5J1XGV@64YZ"O$T\+VXM +MO<0"\/?7I5@U9XDR0)(C+/E@<,9/6F\\RZ8*\9#:H(.I!F!S +MMOYKDE,"0Z"U$KQ6F\+Q+>!:"4HK_11Y*M&+7NS'AG +M^@5T2JBXN6I])@LE:UDGC8VH,$%QH789M>*M^W0X:%W!9>J3_S5'32;%^B%+-<*2' +M*TKJL(C2[OB8F^UYQEL:]F2>W./%QAJ\E-@0B38L&S-LJOV2=0J[&W,.*M,K +MIZ7(*,K**^B,*EVL"OE'.JX#KY^$N4YH0/+0A[RL:.%57DP?"BVJH%+68FD` +MQRRD@C8!EO4]'99'U`/(-#8LD,IB<2=E'@@`'_W+FB4$JXQ0E.@-!T&'72=B +M2>CP%\LTU,L95O=\F*,Y2_TF&?X9X&D]'X]#'/N=T$4%#@*4N,SU@!Q16G<2 +MT%0\#[,^6L&3',""/N:!"&.&7,T]V4GC$/SC*;0-&" +M![,9^<]1X9+87C4RW!R7)J::4:,7,V>;&#_75,/3DI]WY#(T8EX#,(`[,JVA +M&TQ7PB2\_[DR4[DIE;C^3'^L_(*^C=-'/A>["]G_N?K0DP!#CXI9Q%LP#7A) +MTIM;+?_YI52ESKVI6D,C23B+7*>DX6>>=0MWO5CE&;T%JZ>R-A83OKB:"9K)%PW^X9+A;^(7*!W;39&'^ +MF67&/&$)[>(*[5(5K5*"/=% +M]R+E.#A8C19GP5C36/Q97Q4%/2Q@(RB8_[83YDH#_$=GL@X^1YZX&>NCS(K4 +MOM<`M!*6)"@$\YM>R)71$JY[0GNA:F<@0BU=JGN9A9Z,]97OGY/I*Q[:^(!6 +M6YQB-_S3X%,.F$E1:4J.7UTEE@:AD#9AA) +MG@-/M2&-Y"IC/\CC%,G:#230P>T==J=9_8[C!K_.>"W;HT@7K^`+U8M8MI_* +MD=PE3R(ON-+K7E)ECY#$44`*D:CWFQL>[\]&F:.&]"^T^0?:Z/O0V072"H]7 +M!(.&74N6X;O(B4G/U827=++(F;\@FC$)>YZW6\HTJQ4[9=%T]1NTWK<-I`AY +M7Y=AG3]UI*%7M/3U5#H^VDV'!.%[OV*TGRQA*_83@@7YLQGP +MK%#D?FT6SIFL3U,8,!L\[7QRT="VD>6RV-\#%J>'QL1P"3.YCX]K7E4@@B*: +MD`/6UJL*GP43U?V>+94N6;7#9.1MLS$X0X=/=V3ILNHK[ZPX='/>[<2(Y/CF +M$P6KM"Y,_E*>U#GKFEDC`P!Y?"O>.WQO0P(-G)96;-:P_]^DMDB!C0U43R)F +MCTZ/T9&[W8?$!9OFL9QZ2ZA=WTFSMK(S>UMD\R)6+5^(X7$,AX1**V#W'E#9 +MMO+?[M*)*'Y[--U&\RV+>P]K>@7H/4%,>T^%_H!R/7I3:7<@8YI#E_,K`@E0 +M3!4SO*^VELW(XYC%@XI(="H],=AK>,8)71'1]^3@A=O,&(R-83N +MV>/'>-ETD:)H1),.5AYG<"`:"=4X([-N]W"3"D#-D/#*X54(@2&%(#[7J>-F +MPI!0Q)Y4H2`PG;ZPWF6[G1>DKN;>&P[*1^J$)E# +MJ$#0D'O$VT*C*X]Z)U0!YDW\`"W5.7SQ=R?F#N(5*F,5^Q@% +M;@!AG4VPK[8&0K(?"X\JR^2*!]G3H,35G9QGJTAWE2]M"RRE%]9$66I.7'0$ +M8IT1_Z?VH8K5ZGA^KE0$8X]-+'F%USH3%K/K=6#5X +MM%_K#&U2^*:L@]\FTVO#6+P4(Z15!-6?986+D.9IN@PBITHYPN5>FDT!T7+6 +M3BQ]+'T4&U9['(+2*)?QP;^N:_N-?'\K@@V/UV1^_66S43Q(:O%"8W2.*#88 +M1>21!_K(EO)OY_>]$S[%<==I\V!!@\@<%>"GVH-"OS=)GSK?Z?4VHJ/WI>8S +M@<.:R4\&*?+3+-K2I:L3['I8I((H'K")H9.I$19@;@108A)#8'-#6?,6.:\' +M]7F-;I8+6=HF+(X.,XLUY<.1RIOCH)%UP#F2W6TR[DPJS%_8Z.G^J*^FO9W[ +M80"AH+B9IE7QS$(Y$EV`^]B,TR8PW(YVL6@+_3AFH(\"`2MN58)U8#$G>2^! +M@,>IPEDS_^DI$](-3B$<(P6U-F\)Y*Q<6'H.S)9,ZY#/]6T<9-AK4BN"$/UU +M,-.C65&YOXG;\WI5;\Y!7(Y)(D6+V+<64^HR^>;J"2FAQ.6JFW/04"P/V/C0%U:)8 +MX97L[Y/:$JG@9'_^GK5_S%%1X1L>EJ.F?=\G:0A4>BR24GGVT@GM/9[CDO90 +MBG,&AV!P.,GHQY>]5$X.IM3BGB<5:4SLFEM!EQ)]DIFZP!#9(B&=L`U2LG0[ +M]Q!]%,.M.";_EZL<-V=11JV-GI-`V?!M#^S89);U&8AD9U(56H`&)9$_JV%5 +MK:KR.&C)KWO^\X2*(6B\H-HE6XLBQL3`T>D\$%^*.O^; +M1SDSH7!'^P%&RPCBTZ)\[D1ZS'*S2=P2HW%^U^7A)*_)$MI3S]>YU=@V?_YB +M+TM")B?&AR:5'4@+^EE:H,7OP^;:;ZAP]!7@-"M7X&U3=C?Z1]70KF8 +MXYMT\14G]M4)R4S,:WM<%\$0OT4V0K`#Z?=*&1CSI5U +MX;NH5\8OU_^&:?0=NW7/H!\;+`01FWF7I%<[BNBF&IIPN)4$ZVBQT\S+?P=2 +M@E9X6?I+>!'L//-'$)%U,B"VU6G4,TE;71J4S_*]]_@08)MQ?@+PS2]OS^HX +M4#8PT\T+EHHRA(T+>B889=GLM>J_62UHN'GL]R'%*=*$93_NZ+RI0X8NX"61- +MA6`/Q\-UCT^S%TC2,NBL=A>9<"&L!JE:H8#H?"4+YAZOW!RG/V+;+,4,AIEY +MV06I,$/<%P'D^)NL4WF67..(,B<%,L4AHCX21T/CNK1J@(9>V.6P`\4#("I1 +M))$Q4Y0SV*M:]??MY%#U3ITVJ\R-?,]N4NQY)<-P[5*;V.$UK<9D6KAI'EZI +M=1%H&@[Q1_-GF:4IF"P5096G@=6*&:H16=CN"R-%%#`WL)WKN,.\;?<:+Z:# +M3.(YI]E(=*W+O5F.G*B6(8-(6*,*L-E!H_\;B,N$JE(7$*+X&B([A5X76H;% +MM>$SUIK9#G_J,%%0SR;6&.=GR:J*]B:UVK0FA")KLJK%@!$&QM?;@1J#HF\- +M8=#WJB/85U38X)?-37-"4V*USP\TU*00L$_>2E[;)M% +M/@3;][H$A,XGEAXJKL<0K>V`--=N-^1Z-J5P.J46TY!5#'WX9"BC^Y6JT"/` +M/%8LR==37*:@S6!-4-&Y$Y_OJ2\$5AN!A0)0Q<(W;]XR/%9^#&M#K.U0--4@ +M;:E9>A"YQ?ZKZ%9`_CG%C$+T,?29#,,PAN:QP$/TQ`3^^L`$`:;Z5[.1[`W7 +M`@F8=D#Q$NX]Q/D7M*X@@?;7%QBIP'%6=@7XG=.>!C>21+O/B\D;E\>E/"\( +M"6S[>D"Q2-:X\%FGDN-\SXJF4F-79Y@X0Y;\3\&/"S8T=HQK6HD:JQ8S+T$MLNW1S)F@<7E[KRY`GUJNL+(DV$H +MV"!1YO_`//@;^JL!3)I;!#8#+44[6Q161>BN@E"\HF"E%WU57H4;)_V[>O>_ +M@L:3SE>H/%?\3^FILD.T-;>."=3?''_;&.@9,00N&&:'L-4,^+Z8L`.MA[XH +MP66,"C#FV[VANJ5RS$MUOIH;>0ZCN%;71B5:N?.@\V3M?&(LU2CTT7FO##/Y +MEX:%E?EVC`*QO.OG3U8QY9`QVBQG_HO?*B-CW$LENB#@WE<4M^;1GMC3JO)E +M`#0+/0R/.@V&QUJ#.HK80R"/*!1\>J_O%P))5D:\$+9;<'0/]Y'QKM[_R)6J +M'C6(08X*$@1R_B#*LU/('<'1@P^V@KD_[(_U&%I.ME7B()C$DX>L +M??'O[P81Z83(@/<")0-G^.=[BL$K+&<^4U9A^_#YK+#P67KAH(.6H#;H1FAO +M4!,KP&B;A.C-+0=1INM0D'TRY?ZHEI"^LDI!X[;<84JZP@7VI76PG4^E6F7. +MTCHHHZED_Q_F%:C"C=6%7LQBRQX7MZM%8H`67]L\4(VE^?3\2T0,6:R>=%P= +MDZ7\!T4?1@AI&7+%4#<$*92H%V`)71+L8?0 +MHR@JK/\Z.4+"#[&=X;=)#44A7>Z47F_:/ZB?LNL0'KO_';A?O(TNAMVCQ$]) +M2^/TK24LGH)_S$!N,B*9*"G"Y;N0CT`21C!EFL&!"CMA;EHKK_5Q$J(X:)*I +ML?>LH^ZH5C@>]2&=+HP4?1V'$S]+%H"_E8#,S^>;[O[,[.2`-61AFIE9H/@R +M3'(>GYBA*"OL_6#P"M:*0-D`Q&*V:5>V@UZ]SI.FC\[-AQ/?__ +M1&EEZ!,J.*-Y01%SFQ_GC;!3,_["I6CIJ+Z^^2_T72[#]2/SIQ\ZT)7_4I@N +MCVIIZYT,+K13E!FV+"%X,%P`X`H6(X0IU:&`\72Y$7FYE1EQ[_VM!W$=LXI'9+S0!9>'Y'194&GSJ8ZP+.+;M4!N(:EL8"LT,D2)TNYW[ +M!`ZH`LG%%1Z6#SI@"ASIL,ZSM6`,C]3IY2\=C*PQ1J?VB&XBV66SLS:H^R_U +M,Y[^5-P!`#2O*O/3"CM%%UZ&):F_D@0(^'(1LB!"59%7/,*"^3."#?1:W1DF +MC@.*-5RX*G&-U8AFYI?9^KN#B'IA@*5'N[2ZG7.0^[59^=J=->S7L$^>GFOAO;I5N3%$?.J(@? +M&Z.W5MC`L-9JMJ]<+DZNE>M`:(-1.Z1U"1#!*^M&?JFH,F'NQM7M^E9(S`-3)V`E4PT'QBYY:%DE6]+T +M"E.JVA?<,L]!=0F:J:)[J_1IT+MW7M!".*R5<0>:N^K?D90CNI^'"-Y#+[_R,]^!QYB#0XS-)!3[ +MB,,4;MI4R+LWB9=Z)HFBOM4;6^%E7_^R/HRTT0;EJ*2T3&-&:F^?M-&!W@*%!^"'E/$I.;3?Q.ZN-JX^DPXQ\.#)^L,@&2U:]"6&DU9??P +M:\4#,QBZ37Y.;6:YZ6B=7T$C4Y_2`,7$M9F;=%MT>\"P`<7?JS>8'8.J^X/@41+,FC^O\VZT_ +M?#MJZ8N?G:9E<-4BP*ODKZ;;9+6HCEU0#'IV#R]7TS?9W0^RU,2[!POG"^15 +M6.K1`H,3)Z3(->2NHV+;CY08P/Y8>-O#?&Q,6-(2_UZNH@/_IY\$B#P/&\': +M3.[G0^]0P:0K,+V.7PD>6"B+^6K$!H,7*EBEN@AK[%2-0E5_IQK\-:(SY\W> +M=1VX;T%!K!F^?XE2+QOP,IFF<'O-A:$+%XJVXHS<_W +ML:`952%S\^VJX+>@DIUHJGGOQG,)--4SEF^05K2_OIY,]>)QI[%">QK$D*JB +M-IG5A:W.7!HF->ZI"6AAIWE)!353"CV9RZ&:WS:$7*W4[=[5CR9MN;-'DO=BJ-.P#TK<M#>I8A<'_E>^Y/T+"'JYN@QT%AOF1!]OQ]#,63#)"#I^TO +M0#K"LZVBP%48#/;06KC6C3(GDM0,X-(D@,.-,[QM[__-XC&55.:Z"_S;ZC5I +M$RG7*B?EFDQ"[_M.U5-!]]908RJ-H_J\@$<1IEI`_J>)V6UI%&=$IY\)P_&' +MD1^OK$4"I"`J$5*D0Q,),B[`>/^Y/&]!UTT7%3=(5F(6#B'%P]C24=O=UGWM +M_E\6"_4Y+WI&,F_;]O>6Y@1^[J<%^-)E7D!X+=-__HM`4N(6H-,5G;\AT2DI +MRUQBQ::.-I]I4XWC%;`7S'B#.G+^*G4Z#MN7^N[F9Z5\6P.J\,Z18^4Y +M>`S&89511=S* +M%VW1C9KCK_O<4TT`H)!.%:")4F<1/[7A,?$SP1.R6R\P?$7.!XC +MOU9IA!F,/$;$&A(+_D";A$3N*;OH&:QNEJT+AVXHI9:6KJ52I;O6<4@2JM'2 +MPBD1YF9`75?YRU8*IO1TSS0R.[V0NS+8Y9H$Q%.62#66>47QDWVLEP*PF=+4/GZ95W +MLB.LXA_DVX!1"WFO7WI +M$@DYYSJ>;4)0#G__5L?S>,HXDV"4>+M@9J$S]ZGK_UG:T0&;;Y_(!LE8PA!0 +M3%T0`!]+LT"%-9^L6-(R;$="_0(!"AE;^4:J3EDLFUF^,K:?._Q?\ZL;C&\S +MD&\'#&G^5N<,]:4"JU5,LN!1IM1*V\2B7@(]J0X"X +MIIG_U&(9I:L`M?I&^#^*\%[9XW*5F+#(VA>`H<"MPKNFRG''XBT0MYVJ6>'D +MEN>C+%;C=L9G,B>.",!80S<0L754:^M@%GO*]M]9&'LP-+OBJ5DZ\0AD/>,Z +MZS%I/N%&,TU>2X`,5PFZ\KO=F)P03;IO$!Y(L<1G^P(`````!%E: +` +end diff --git a/downloader.sh b/downloader.sh new file mode 100755 index 0000000..ebc2b86 --- /dev/null +++ b/downloader.sh @@ -0,0 +1,124 @@ +#!/bin/bash +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +set -euo pipefail + +download_file() { + local DOWNLOAD_PATH="${1}" + local DOWNLOAD_URL="${2}" + for (( COUNTER=0; COUNTER<=60; COUNTER+=1 )); do + if [ ${COUNTER} -ge 1 ] ; then + sleep 1s + fi + set +e + curl ${CURL_SSL:-} --fail --location -o "${DOWNLOAD_PATH}" "${DOWNLOAD_URL}" &> /dev/null + local CMD_RET=$? + set -e + + case ${CMD_RET} in + 0) + export CURL_SSL="${CURL_SSL:-}" + return + ;; + 60) + if [ -f /etc/ssl/certs/cacert.pem ] ; then + export CURL_SSL="--cacert /etc/ssl/certs/cacert.pem" + continue + fi + + set +e + dialog --keep-window --title "Bad Certificates" --defaultno \ + --yesno "CA certificates need to be fixed, do you want me to fix them?\n\nNOTE: This operation will delete files at /etc/ssl/certs" \ + 7 65 + local DIALOG_RET=$? + set -e + + if [[ "${DIALOG_RET}" == "0" ]] ; then + local RO_ROOT="false" + if mount | grep "on / .*[(,]ro[,$]" -q ; then + RO_ROOT="true" + fi + [ "${RO_ROOT}" == "true" ] && mount / -o remount,rw + rm /etc/ssl/certs/* 2> /dev/null || true + echo + echo "https://curl.se/ca/cacert.pem" + curl -kL "https://curl.se/ca/cacert.pem"|awk 'split_after==1{n++;split_after=0} /-----END CERTIFICATE-----/ {split_after=1} {if(length($0) > 0) print > "/etc/ssl/certs/cert" n ".pem"}' + echo + echo "Installing cacert.pem into /etc/ssl/certs ..." + for PEM in /etc/ssl/certs/*.pem; do mv "$PEM" "$(dirname "$PEM")/$(cat "$PEM" | grep -m 1 '^[^#]').pem"; done + for PEM in /etc/ssl/certs/*.pem; do for HASH in $(openssl x509 -subject_hash_old -hash -noout -in "$PEM" 2>/dev/null); do ln -s "$(basename "$PEM")" "$(dirname "$PEM")/$HASH.0"; done; done + sync + [ "${RO_ROOT}" == "true" ] && mount / -o remount,ro + echo + echo "CA certificates have been successfully fixed." + export CURL_SSL="" + continue + fi + + set +e + dialog --keep-window --title "Insecure Connection" --defaultno \ + --yesno "Would you like to run this tool using an insecure connection?\n\nNOTE: You should fix the certificates instead." \ + 7 67 + DIALOG_RET=$? + set -e + + if [[ "${DIALOG_RET}" == "0" ]] ; then + echo + echo "WARNING! Connection is insecure." + export CURL_SSL="--insecure" + sleep 5s + echo + continue + fi + + echo "No secure connection is possible without fixing the certificates." + exit 1 + ;; + *) + echo "No Internet connection, please try again later." + exit 1 + ;; + esac + done + + echo "Internet connection failed, please try again later." + exit 1 +} + +echo "Running MiSTer Downloader" +echo + +SCRIPT_PATH="/tmp/downloader.sh" + +rm ${SCRIPT_PATH} 2> /dev/null || true + +download_file "${SCRIPT_PATH}" "https://raw.githubusercontent.com/MiSTer-devel/Downloader_MiSTer/main/dont_download.sh" + +chmod +x "${SCRIPT_PATH}" + +export DOWNLOADER_LAUNCHER_PATH="${BASH_SOURCE[0]}" + +if ! "${SCRIPT_PATH}" ; then + echo -e "Downloader failed!\n" + exit 1 +fi + +rm ${SCRIPT_PATH} 2> /dev/null || true + +exit 0 diff --git a/src/__main__.py b/src/__main__.py new file mode 100755 index 0000000..5743b7b --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import os +from downloader.main import main + +if __name__ == '__main__': + exit_code = main({ + 'DOWNLOADER_LAUNCHER_PATH': os.getenv('DOWNLOADER_LAUNCHER_PATH', None), + 'CURL_SSL': os.getenv('CURL_SSL', '--cacert /etc/ssl/certs/cacert.pem'), + 'COMMIT': os.getenv('COMMIT', 'unknown'), + 'ALLOW_REBOOT': os.getenv('ALLOW_REBOOT', None), + 'UPDATE_LINUX': os.getenv('UPDATE_LINUX', 'true'), + 'DEFAULT_DB_URL': os.getenv('DEFAULT_DB_URL', 'https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/db.json.zip'), + 'DEFAULT_DB_ID': os.getenv('DEFAULT_DB_ID', 'distribution_mister') + }) + + exit(exit_code) diff --git a/src/automated_tests.sh b/src/automated_tests.sh new file mode 100755 index 0000000..e52339c --- /dev/null +++ b/src/automated_tests.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Copyright (c) 2021 José Manuel Barroso Galindo + +set -euo pipefail + +cd src +echo "Unit Tests:" +python3 -m unittest discover -s test/unit +echo +echo "Integration Tests:" +python3 -m unittest discover -s test/integration diff --git a/src/build.sh b/src/build.sh new file mode 100755 index 0000000..e8c2192 --- /dev/null +++ b/src/build.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Copyright (c) 2021 José Manuel Barroso Galindo + +set -euo pipefail + +TEMP_ZIP1="$(mktemp -u).zip" +TEMP_ZIP2="$(mktemp -u).zip" +BIN="/tmp/dont_download.zip" +COMMIT="$(git rev-parse --short HEAD)" + +pin_metadata() { + touch -a -m -t 202108231405 "${1}" +} + +cd src + +find downloader -type f -iname "*.py" -print0 | while IFS= read -r -d '' file ; do pin_metadata "${file}" ; done +pin_metadata __main__.py +zip -q -0 -D -X -A "${TEMP_ZIP1}" __main__.py downloader/*.py +pin_metadata "${TEMP_ZIP1}" +echo '#!/usr/bin/env python3' | cat - "${TEMP_ZIP1}" > "${TEMP_ZIP2}" +pin_metadata "${TEMP_ZIP2}" +rm "${TEMP_ZIP1}" +cd .. + +cat <<-EOF +#!/usr/bin/env bash +set -euo pipefail +export DOWNLOADER_LAUNCHER_PATH="\${DOWNLOADER_LAUNCHER_PATH:-\${0}}" +export COMMIT="${COMMIT}" +uudecode -o - "\${0}" | xzcat -d -c > "${BIN}" +chmod a+x "${BIN}" +"${BIN}" +exit 0 +EOF + +uuencode - < <(xzcat -z < "${TEMP_ZIP2}") +rm "${TEMP_ZIP2}" > /dev/null 2>&1 || true diff --git a/src/debug.sh b/src/debug.sh new file mode 100755 index 0000000..3aabc11 --- /dev/null +++ b/src/debug.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Copyright (c) 2021 José Manuel Barroso Galindo + +set -euo pipefail + +MISTER_IP=${MISTER_IP:-$(cat "mister.ip" | tr -d '[:space:]')} +TEMP_SCRIPT="$(mktemp)" + +./src/build.sh > "${TEMP_SCRIPT}" +chmod +x "${TEMP_SCRIPT}" + +if [ -f dont_download.ini ] ; then + sshpass -p 1 scp -o StrictHostKeyChecking=no dont_download.ini "root@${MISTER_IP}:/media/fat/downloader.ini" +fi +sshpass -p 1 scp -o StrictHostKeyChecking=no "${TEMP_SCRIPT}" "root@${MISTER_IP}:/media/fat/downloader.sh" +sshpass -p 1 scp -o StrictHostKeyChecking=no downloader.sh "root@${MISTER_IP}:/media/fat/Scripts/downloader.sh" +rm "${TEMP_SCRIPT}" + +echo "OK" diff --git a/src/downloader/__init__.py b/src/downloader/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/downloader/config.py b/src/downloader/config.py new file mode 100644 index 0000000..277b072 --- /dev/null +++ b/src/downloader/config.py @@ -0,0 +1,132 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import configparser +from enum import IntEnum, unique +from pathlib import Path, PurePosixPath +from .ini_parser import IniParser + + +def config_file_path(original_executable): + if original_executable is None: + return '/media/fat/downloader.ini' + + executable_path = PurePosixPath(original_executable) + parent = str(executable_path.parent) + + if original_executable[0] == '/' and executable_path.parents[0].name.lower() == 'scripts': + list_of_parents = [str(p.name) for p in reversed(executable_path.parents)] + parent = '/'.join(list_of_parents[0:-1]) + + return parent + '/' + executable_path.stem + '.ini' + + +@unique +class AllowDelete(IntEnum): + NONE = 0 + ALL = 1 + OLD_RBF = 2 + + +@unique +class AllowReboot(IntEnum): + NEVER = 0 + ALWAYS = 1 + ONLY_AFTER_LINUX_UPDATE = 2 + + +def default_config(): + return { + 'databases': [], + 'base_path': '/media/fat/', + 'base_system_path': '/media/fat/', + 'allow_delete': AllowDelete.ALL, + 'allow_reboot': AllowReboot.ALWAYS, + 'check_manually_deleted_files': True, + 'update_linux': True, + 'parallel_update': True, + 'downloader_size_mb_limit': 100, + 'downloader_process_limit': 300, + 'downloader_timeout': 300, + 'downloader_retries': 3, + } + + +class ConfigReader: + def __init__(self, logger, env): + self._logger = logger + self._env = env + + def default_db_config(self): + return { + 'db_url': self._env['DEFAULT_DB_URL'], + 'section': self._env['DEFAULT_DB_ID'] + } + + def read_config(self, config_path): + result = default_config() + result['config_path'] = Path(config_path) + + default_db = self.default_db_config() + + ini_config = configparser.ConfigParser(inline_comment_prefixes=(';', '#')) + try: + ini_config.read(config_path) + except Exception as e: + self._logger.debug(e) + self._logger.print('Could not read ini file %s' % config_path) + + for section in ini_config.sections(): + parser = IniParser(ini_config[section]) + if section.lower() == 'mister': + result['base_path'] = parser.get_string('base_path', result['base_path']) + result['base_system_path'] = parser.get_string('base_system_path', result['base_system_path']) + result['allow_delete'] = AllowDelete(parser.get_int('allow_delete', result['allow_delete'].value)) + result['allow_reboot'] = AllowReboot(parser.get_int('allow_reboot', result['allow_reboot'].value)) + result['check_manually_deleted_files'] = parser.get_bool('check_manually_deleted_files', result['check_manually_deleted_files']) + result['parallel_update'] = parser.get_bool('parallel_update', result['parallel_update']) + result['update_linux'] = parser.get_bool('update_linux', result['update_linux']) + result['downloader_size_mb_limit'] = parser.get_int('downloader_size_mb_limit', + result['downloader_size_mb_limit']) + result['downloader_process_limit'] = parser.get_int('downloader_process_limit', + result['downloader_process_limit']) + result['downloader_timeout'] = parser.get_int('downloader_timeout', result['downloader_timeout']) + result['downloader_retries'] = parser.get_int('downloader_retries', result['downloader_retries']) + continue + + self._logger.print("Reading '%s' db section" % section) + default_db_url = default_db['db_url'] if section.lower() == default_db['section'].lower() else None + db_url = parser.get_string('db_url', default_db_url) + if db_url is None: + raise Exception("Can't import db for section '%s' without an url field" % section) + result['databases'].append({ + 'db_url': db_url, + 'section': section + }) + + if len(result['databases']) == 0: + self._logger.print('Reading default db') + result['databases'].append({ + 'db_url': ini_config['DEFAULT'].get('db_url', default_db['db_url']), + 'section': default_db['section'] + }) + + if self._env['ALLOW_REBOOT'] is not None: + result['allow_reboot'] = AllowReboot(int(self._env['ALLOW_REBOOT'])) + + return result diff --git a/src/downloader/curl_downloader.py b/src/downloader/curl_downloader.py new file mode 100644 index 0000000..a944a94 --- /dev/null +++ b/src/downloader/curl_downloader.py @@ -0,0 +1,237 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import shlex +import subprocess +import sys +from urllib.parse import quote, urlparse +import urllib.request +import time + + +def make_downloader_factory(file_service, logger): + return lambda config: CurlCustomParallelDownloader(config, file_service, logger) if config[ + 'parallel_update'] else CurlSerialDownloader(config, file_service, logger) + + +class CurlCommonDownloader: + def __init__(self, config, file_service, logger): + self._config = config + self._file_service = file_service + self._logger = logger + self._curl_list = {} + self._errors = [] + self._http_oks = [] + self._correct_downloads = [] + self._needs_reboot = False + + def queue_file(self, file_description, file_path): + self._curl_list[file_path] = file_description + + def download_files(self, first_run): + self._download_files_internal(first_run) + + if self._file_service.is_file('MiSTer.new'): + self._logger.print() + self._logger.print('Copying new MiSTer binary:') + self._file_service.unlink('MiSTer') + self._file_service.move('MiSTer.new', 'MiSTer') + + if self._file_service.is_file('MiSTer'): + self._logger.print('New MiSTer binary copied.') + else: + # This error message should never happen. + # If it happens it would be an unexpected case where file_service is not moving files correctly + self._logger.print('CRITICAL ERROR!!! Could not restore the MiSTer binary!') + self._logger.print('Please manually rename the file MiSTer.new as MiSTer') + self._logger.print('Your system won\'nt be able to boot until you do so!') + sys.exit(1) + + def _download_files_internal(self, first_run): + if len(self._curl_list) == 0: + self._logger.print("Nothing new to download from given sources.") + return + + self._logger.print("Downloading %d files:" % len(self._curl_list)) + + for path in sorted(self._curl_list): + if 'path' in self._curl_list[path] and self._curl_list[path]['path'] == 'system': + self._file_service.add_system_path(path) + + if self._file_service.is_file(path): + path_hash = self._file_service.hash(path) + if path_hash == self._curl_list[path]['hash']: + self._logger.print('No changes: %s' % path) + self._correct_downloads.append(path) + continue + else: + self._logger.debug('%s: %s != %s' % (path, self._curl_list[path]['hash'], path_hash)) + + if first_run: + for delete in self._curl_list[path]['delete']: + self._file_service.clean_expression(delete) + + self._download(path, self._curl_list[path]) + + self._wait() + + for retry in range(self._config['downloader_retries']): + if len(self._http_oks) > 0: + self._logger.print() + self._logger.print('Checking hashes...') + + for path in self._http_oks: + if not self._file_service.is_file(path if path != 'MiSTer' else 'MiSTer.new'): + self._logger.print('Missing %s' % path) + self._errors.append(path) + continue + + path_hash = self._file_service.hash(path if path != 'MiSTer' else 'MiSTer.new') + if path_hash != self._curl_list[path]['hash']: + self._logger.print( + 'Bad hash on %s (%s != %s)' % (path, self._curl_list[path]['hash'], path_hash)) + self._errors.append(path) + continue + + self._logger.print('+', end='', flush=True) + self._correct_downloads.append(path) + if self._curl_list[path].get('reboot', False): + self._needs_reboot = True + + self._logger.print() + + self._http_oks = [] + + if len(self._errors) == 0: + return + + missing = self._errors + self._errors = [] + + for path in missing: + self._download(path, self._curl_list[path]) + + self._wait() + + def _download(self, path, description): + self._logger.print(path) + self._file_service.makedirs_parent(path) + + url_domain = urlparse(description['url']).netloc + url_parts = description['url'].split(url_domain) + + target_path = self._file_service.curl_target_path(path if path != 'MiSTer' else 'MiSTer.new') + url = url_parts[0] + url_domain + urllib.parse.quote(url_parts[1]) + + self._run(description, self._command(target_path, url), path) + + def _command(self, target_path, url): + return 'curl %s --show-error --fail --location -o "%s" "%s"' % (self._config['curl_ssl'], target_path, url) + + def errors(self): + return self._errors + + def correctly_downloaded_files(self): + return self._correct_downloads + + def needs_reboot(self): + return self._needs_reboot + + def _wait(self): + raise NotImplementedError() + + def _run(self, description, command, path): + raise NotImplementedError() + + +class CurlCustomParallelDownloader(CurlCommonDownloader): + def __init__(self, config, file_service, logger): + super().__init__(config, file_service, logger) + self._processes = [] + self._files = [] + self._acc_size = 0 + + def _run(self, description, command, file): + self._acc_size = self._acc_size + description['size'] + + result = subprocess.Popen(shlex.split(command), shell=False, stderr=subprocess.DEVNULL, + stdout=subprocess.DEVNULL) + + self._processes.append(result) + self._files.append(file) + + if self._acc_size > (1000 * 1000 * self._config['downloader_size_mb_limit']) or len(self._processes) > \ + self._config['downloader_process_limit']: + self._wait() + + def _wait(self): + count = 0 + start = time.time() + while count < len(self._processes): + some_completed = False + for i, p in enumerate(self._processes): + if p is None: + continue + result = p.poll() + if result is not None: + self._processes[i] = None + some_completed = True + count = count + 1 + start = time.time() + self._logger.print('.', end='', flush=True) + if result == 0: + self._http_oks.append(self._files[i]) + else: + self._logger.print('~', end='', flush=True) + self._logger.debug('Bad http code! %s: %s' % (result, self._files[i]), flush=True) + self._errors.append(self._files[i]) + end = time.time() + if (end - start) > self._config['downloader_timeout']: + for i, p in enumerate(self._processes): + if p is None: + continue + self._errors.append(self._files[i]) + self._logger.print('Timeout! %s' % self._files[i], flush=True) + break + + time.sleep(1) + if not some_completed: + self._logger.print('*', end='', flush=True) + + self._logger.print(flush=True) + self._processes = [] + self._files = [] + self._acc_size = 0 + + +class CurlSerialDownloader(CurlCommonDownloader): + def __init__(self, config, file_service, logger): + super().__init__(config, file_service, logger) + + def _run(self, description, command, file): + result = subprocess.run(shlex.split(command), shell=False, stderr=subprocess.STDOUT) + if result.returncode == 0: + self._http_oks.append(file) + else: + self._logger.print('Bad http code! %s: %s' % (result.returncode, file), flush=True) + self._errors.append(file) + + self._logger.print() + + def _wait(self): + pass diff --git a/src/downloader/db_gateway.py b/src/downloader/db_gateway.py new file mode 100644 index 0000000..f90929a --- /dev/null +++ b/src/downloader/db_gateway.py @@ -0,0 +1,46 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import tempfile +from pathlib import Path +from .other import run_successfully + + +class DbGateway: + def __init__(self, config, file_service, logger): + self._config = config + self._file_service = file_service + self._logger = logger + + def fetch(self, db_uri): + if not db_uri.startswith("http"): + if not db_uri.startswith("/"): + db_uri = str(Path(db_uri).resolve()) + return self._file_service.load_db_from_file(db_uri) + + try: + with tempfile.NamedTemporaryFile(delete=False) as tmp_file: + run_successfully('curl %s --silent --show-error --fail --location -o %s %s' % ( + self._config['curl_ssl'], tmp_file.name, db_uri), self._logger) + + return self._file_service.load_db_from_file(tmp_file.name, Path(db_uri).suffix.lower()) + + except Exception as e: + self._logger.debug(e) + self._logger.print('Could not load json from "%s"' % db_uri) + return None \ No newline at end of file diff --git a/src/downloader/file_service.py b/src/downloader/file_service.py new file mode 100644 index 0000000..ac73e9e --- /dev/null +++ b/src/downloader/file_service.py @@ -0,0 +1,148 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import os +import hashlib +from pathlib import Path +import shutil +import json +from .other import run_stdout, run_successfully +from .config import AllowDelete +import subprocess + + +class FileService: + def __init__(self, config, logger): + self._config = config + self._logger = logger + self._system_paths = set() + + def add_system_path(self, path): + self._system_paths.add(path) + + def is_file(self, path): + return os.path.isfile(self._path(path)) + + def read_file_contents(self, path): + with open(self._path(path), 'r') as f: + return f.read() + + def write_file_contents(self, path, content): + with open(self._path(path), 'w') as f: + return f.write(content) + + def touch(self, path): + return Path(self._path(path)).touch() + + def move(self, source, target): + self.copy(source, target) + self._unlink(source, False) + + def copy(self, source, target): + return shutil.copyfile(self._path(source), self._path(target)) + + def hash(self, path): + return hash_file(self._path(path)) + + def makedirs(self, path): + return os.makedirs(self._path(path), exist_ok=True) + + def makedirs_parent(self, path): + return os.makedirs(str(Path(self._path(path)).parent), exist_ok=True) + + def curl_target_path(self, path): + return self._path(path) + + def unlink(self, path): + if self._config['allow_delete'] != AllowDelete.ALL: + if self._config['allow_delete'] == AllowDelete.OLD_RBF and ( + path[-4:].lower() == ".rbf" or path == 'MiSTer'): + return self._unlink(path, True) + + return True + + return self._unlink(path, True) + + def clean_expression(self, expr): + if self._config['allow_delete'] != AllowDelete.ALL: + return True + + if expr[-1:] == '*': + self._logger.debug('Cleaning %s ' % Path(expr).name, end='') + result = subprocess.run('rm "%s"*' % self._path(expr[0: -1]), shell=True, stderr=subprocess.DEVNULL) + return result.returncode == 0 + else: + return self._unlink(expr, True) + + def load_db_from_file(self, path, suffix=None): + path = self._path(path) + if suffix is None: + suffix = Path(path).suffix.lower() + if suffix == '.json': + return self._load_json(path) + elif suffix == '.zip': + return self._load_json_from_zip(path) + else: + raise Exception('File type "%s" not supported' % suffix) + + def _load_json_from_zip(self, path): + json_str = run_stdout("unzip -p %s" % path) + return json.loads(json_str) + + def _load_json(self, file_path): + with open(file_path, "r") as f: + return json.loads(f.read()) + + def save_json_on_zip(self, db, path): + json_name = Path(path).stem + json_path = '/tmp/%s' % json_name + with open(json_path, 'w') as f: + json.dump(db, f) + + zip_path = Path(self._path(path)).absolute() + + run_successfully('cd /tmp/ && zip -qr -9 %s %s' % (zip_path, json_name), self._logger) + + self._unlink(json_path, False) + + def _unlink(self, path, verbose): + if verbose: + self._logger.print('Removing %s' % path) + try: + Path(self._path(path)).unlink() + return True + except Exception as _: + return False + + def _path(self, path): + if path[0] == '/': + return path + + base_path = self._config['base_system_path'] if path in self._system_paths else self._config['base_path'] + + return '%s/%s' % (base_path, path) + + +def hash_file(path): + with open(path, "rb") as f: + file_hash = hashlib.md5() + chunk = f.read(8192) + while chunk: + file_hash.update(chunk) + chunk = f.read(8192) + return file_hash.hexdigest() diff --git a/src/downloader/ini_parser.py b/src/downloader/ini_parser.py new file mode 100644 index 0000000..7444cfe --- /dev/null +++ b/src/downloader/ini_parser.py @@ -0,0 +1,64 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import distutils +import distutils.util + + +class IniParser: + def __init__(self, ini_args): + self._ini_args = ini_args + + def get_string(self, key, default): + result = self._ini_args.get(key, default) + if result is None: + return None + return self._ini_args.get(key, default).strip('"\' ') + + def get_bool(self, key, default): + return bool(distutils.util.strtobool(self.get_string(key, 'true' if default else 'false'))) + + def get_int(self, key, default): + result = self.get_string(key, None) + if result is None: + return default + + return to_int(result, default) + + def get_str_list(self, key, default): + result = [s for s in [s.strip('"\' ') for s in self.get_string(key, '')] if s != ''] + if len(result) > 0: + return result + else: + return default + + def get_int_list(self, key, default): + result = [s for s in [to_int(s, None) for s in self.get_str_list(key, [])] if s is not None] + if len(result) > 0: + return result + else: + return default + + +def to_int(n, default): + try: + return int(n) + except Exception as _: + if isinstance(default, Exception): + raise default + return default diff --git a/src/downloader/linux_updater.py b/src/downloader/linux_updater.py new file mode 100644 index 0000000..5824d06 --- /dev/null +++ b/src/downloader/linux_updater.py @@ -0,0 +1,167 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import subprocess + + +class LinuxUpdater: + def __init__(self, file_service, downloader, logger): + self._downloader = downloader + self._logger = logger + self._file_service = file_service + self._linux_descriptions = [] + + def add_db(self, db): + if 'linux' in db: + self._linux_descriptions.append({ + 'id': db['db_id'], + 'args': db['linux'] + }) + + def update_linux(self): + + linux_descriptions_count = len(self._linux_descriptions) + if linux_descriptions_count == 0: + return + + if linux_descriptions_count > 1: + self._logger.print('Too many databases try to update linux.') + self._logger.print('Only 1 can be processed.') + self._logger.print('Ignoring:') + for ignored in self._linux_descriptions[1:]: + self._logger.print(' - %s' % ignored['id']) + self._logger.print() + + description = self._linux_descriptions[0] + + linux = description['args'] + linux_path = 'linux.7z' + + current_linux_version = 'unknown' + if self._file_service.is_file('/MiSTer.version'): + current_linux_version = self._file_service.read_file_contents('/MiSTer.version') + + if current_linux_version == linux['version'][-6:]: + return + + self._logger.print('Linux will be updated from %s:' % description['id']) + self._logger.print('Current linux version -> %s' % current_linux_version) + self._logger.print('Latest linux version -> %s' % linux['version'][-6:]) + self._logger.print() + + self._downloader.queue_file(linux, linux_path) + if not self._file_service.is_file('/media/fat/linux/7za'): + self._downloader.queue_file({ + 'delete': [], + 'url': 'https://github.com/MiSTer-devel/SD-Installer-Win64_MiSTer/raw/master/7za.gz', + 'hash': 'ed1ad5185fbede55cd7fd506b3c6c699', + 'size': 465600 + }, '/media/fat/linux/7za.gz') + + self._downloader.download_files(False) + self._logger.print() + + if len(self._downloader.errors()) > 0: + self._logger.print('Some error happened during the Linux download:') + for error in self._downloader.errors(): + self._logger.print(error) + + self._logger.print() + return + + self._run_subprocesses(linux, linux_path) + + def _run_subprocesses(self, linux, linux_path): + if self._file_service.is_file('/media/fat/linux/7za.gz'): + result = subprocess.run('gunzip "/media/fat/linux/7za.gz"', shell=True, stderr=subprocess.STDOUT) + self._file_service.unlink('/media/fat/linux/7za.gz') + if result.returncode != 0: + self._logger.print('ERROR! Could not install 7z.') + self._logger.print('Error code: %d' % result.returncode) + self._logger.print() + return + + if not self._file_service.is_file('/media/fat/linux/7za'): + self._logger.print('ERROR! 7z is not present in the system.') + self._logger.print('Aborting Linux update.') + self._logger.print() + return + + result = subprocess.run(''' + sync + RET_CODE= + if /media/fat/linux/7za t "{0}" ; then + if [ -d /media/fat/linux.update ] + then + rm -R "/media/fat/linux.update" > /dev/null 2>&1 + fi + mkdir "/media/fat/linux.update" + if /media/fat/linux/7za x -y "{0}" files/linux/* -o"/media/fat/linux.update" ; then + RET_CODE=0 + else + rm -R "/media/fat/linux.update" > /dev/null 2>&1 + sync + touch /tmp/downloader_needs_reboot_after_linux_update + RET_CODE=101 + fi + else + echo "Downloaded installer 7z is broken, deleting {0}" + RET_CODE=102 + fi + rm "{0}" > /dev/null 2>&1 + exit $RET_CODE + '''.format(self._file_service.curl_target_path(linux_path)), shell=True, stderr=subprocess.STDOUT) + + if result.returncode != 0: + self._logger.print('ERROR! Could not uncompress the linux installer.') + self._logger.print('Error code: %d' % result.returncode) + self._logger.print() + return + + self._logger.print() + self._logger.print("======================================================================================") + self._logger.print("Hold your breath: updating the Kernel, the Linux filesystem, the bootloader and stuff.") + self._logger.print("Stopping this will make your SD unbootable!") + self._logger.print() + self._logger.print("If something goes wrong, please download the SD Installer from") + self._logger.print(linux['url']) + self._logger.print("and copy the content of the files/linux/ directory in the linux directory of the SD.") + self._logger.print("Reflash the bootloader with the SD Installer if needed.") + self._logger.print("======================================================================================") + self._logger.print() + + result = subprocess.run(''' + sync + mv -f "/media/fat/linux.update/files/linux/linux.img" "/media/fat/linux/linux.img.new" + mv -f "/media/fat/linux.update/files/linux/"* "/media/fat/linux/" + rm -R "/media/fat/linux.update" > /dev/null 2>&1 + sync + /media/fat/linux/updateboot + sync + mv -f "/media/fat/linux/linux.img.new" "/media/fat/linux/linux.img" + sync + touch /tmp/downloader_needs_reboot_after_linux_update + ''', shell=True, stderr=subprocess.STDOUT) + + if result.returncode != 0: + self._logger.print('ERROR! Something went wrong during the Linux update, try again later.') + self._logger.print('Error code: %d' % result.returncode) + self._logger.print() + + def needs_reboot(self): + return self._file_service.is_file('/tmp/downloader_needs_reboot_after_linux_update') diff --git a/src/downloader/local_repository.py b/src/downloader/local_repository.py new file mode 100644 index 0000000..31ebcaa --- /dev/null +++ b/src/downloader/local_repository.py @@ -0,0 +1,69 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +class LocalRepository: + def __init__(self, config, logger, file_service): + self._config = config + self._logger = logger + self._file_service = file_service + self._storage_path_value = None + self._last_successful_run_value = None + self._logfile_path_value = None + + @property + def _storage_path(self): + if self._storage_path_value is None: + self._storage_path_value = 'Scripts/.config/downloader/%s.json.zip' % self._config['config_path'].stem + self._file_service.add_system_path(self._storage_path_value) + return self._storage_path_value + + @property + def _last_successful_run(self): + if self._last_successful_run_value is None: + self._last_successful_run_value = 'Scripts/.config/downloader/%s.last_successful_run' % self._config[ + 'config_path'].stem + self._file_service.add_system_path(self._last_successful_run_value) + return self._last_successful_run_value + + @property + def logfile_path(self): + if self._logfile_path_value is None: + self._logfile_path_value = 'Scripts/.config/downloader/%s.log' % self._config['config_path'].stem + self._file_service.add_system_path(self._logfile_path_value) + return self._logfile_path_value + + def load_store(self): + try: + if self._file_service.is_file(self._storage_path): + return self._file_service.load_db_from_file(self._storage_path) + except Exception as e: + self._logger.debug(e) + self._logger.print('Could not load storage') + + return {} + + def has_last_successful_run(self): + return self._file_service.is_file(self._last_successful_run) + + def save_store(self, local_store): + self._file_service.makedirs_parent(self._storage_path) + self._file_service.save_json_on_zip(local_store, self._storage_path) + self._file_service.touch(self._last_successful_run) + + def save_log_from_tmp(self, path): + self._file_service.move(path, self.logfile_path) diff --git a/src/downloader/logger.py b/src/downloader/logger.py new file mode 100644 index 0000000..bb50dcd --- /dev/null +++ b/src/downloader/logger.py @@ -0,0 +1,44 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import tempfile +import sys + + +class Logger: + def __init__(self): + self._logfile = tempfile.NamedTemporaryFile('w', delete=False) + self._local_repository = None + + def set_local_repository(self, local_repository): + self._local_repository = local_repository + + def close_logfile(self): + if self._local_repository is not None: + self._logfile.close() + self._local_repository.save_log_from_tmp(self._logfile.name) + self._logfile = None + self._local_repository = None + + def print(self, *args, sep='', end='\n', file=sys.stdout, flush=True): + print(*args, sep=sep, end=end, file=file, flush=flush) + self.debug(*args, sep=sep, end=end, flush=flush) + + def debug(self, *args, sep='', end='\n', flush=True): + if self._logfile is not None: + print(*args, sep=sep, end=end, file=self._logfile, flush=flush) diff --git a/src/downloader/main.py b/src/downloader/main.py new file mode 100644 index 0000000..ff945a8 --- /dev/null +++ b/src/downloader/main.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import time +import datetime +import subprocess +import traceback + +from .config import config_file_path, ConfigReader +from .online_importer import OnlineImporter +from .logger import Logger +from .curl_downloader import make_downloader_factory, CurlSerialDownloader +from .local_repository import LocalRepository +from .linux_updater import LinuxUpdater +from .reboot_calculator import RebootCalculator +from .offline_importer import OfflineImporter +from .file_service import FileService +from .db_gateway import DbGateway +from .other import format_files_message, empty_store + + +def main(env): + logger = Logger() + try: + exit_code = main_internal(env, logger) + except Exception as _: + logger.print(traceback.format_exc()) + exit_code = 1 + + logger.close_logfile() + return exit_code + + +def main_internal(env, logger): + logger.print('START!') + + ini_path = config_file_path(env['DOWNLOADER_LAUNCHER_PATH']) + + logger.print() + logger.print("Reading file: %s" % ini_path) + + config = ConfigReader(logger, env).read_config(ini_path) + config['curl_ssl'] = env['CURL_SSL'] + + file_service = FileService(config, logger) + local_repository = LocalRepository(config, logger, file_service) + + logger.set_local_repository(local_repository) + + db_gateway = DbGateway(config, file_service, logger) + offline_importer = OfflineImporter(config, file_service, logger) + online_importer = OnlineImporter(config, file_service, make_downloader_factory(file_service, logger), logger) + linux_updater = LinuxUpdater(file_service, CurlSerialDownloader(config, file_service, logger), logger) + + exit_code = run_downloader( + env, + config, + logger, + local_repository, + db_gateway, + offline_importer, + online_importer, + linux_updater + ) + + needs_reboot = RebootCalculator(config, logger, file_service).calc_needs_reboot( + linux_updater.needs_reboot(), + online_importer.needs_reboot()) + + if needs_reboot: + logger.print() + logger.print("Rebooting in 10 seconds...") + time.sleep(5) + logger.close_logfile() + time.sleep(5) + subprocess.run(['reboot', 'now'], shell=False, stderr=subprocess.STDOUT) + + return exit_code + + +def run_downloader(env, config, logger, local_repository, db_gateway, offline_importer, online_importer, linux_updater): + start = time.time() + + local_store = local_repository.load_store() + failed_dbs = [] + + for db_description in config['databases']: + + db = db_gateway.fetch(db_description['db_url']) + if db is None: + failed_dbs.append(db_description['db_url']) + continue + + if db['db_id'] != db_description['section']: + failed_dbs.append(db_description['db_url']) + logger.print('Section %s doesn\'t match database id "%s"' % (db_description['section'], db['db_id'])) + continue + + if db['db_id'] not in local_store: + local_store[db['db_id']] = empty_store() + + store = local_store[db['db_id']] + + offline_importer.add_db(db, store) + online_importer.add_db(db, store) + linux_updater.add_db(db) + + if env['UPDATE_LINUX'] != 'only': + + offline_importer.apply_offline_databases() + full_resync = not local_repository.has_last_successful_run() + online_importer.download_dbs_contents(full_resync) + + run_time = str(datetime.timedelta(seconds=time.time() - start))[0:-4] + + logger.print() + logger.print('===========================') + logger.print('Downloader 1.0 (%s) by theypsilon. Run time: %ss' % (env['COMMIT'], run_time)) + logger.print('Log: %s' % local_repository.logfile_path) + logger.print() + logger.print('Installed:') + logger.print(format_files_message(online_importer.correctly_installed_files())) + logger.print() + + logger.print('Errors:') + logger.print(format_files_message(online_importer.files_that_failed() + failed_dbs)) + + logger.print() + local_repository.save_store(local_store) + + if env['UPDATE_LINUX'] != 'false' and config.get('update_linux', True): + linux_updater.update_linux() + if env['UPDATE_LINUX'] == 'only' and not linux_updater.needs_reboot(): + logger.print('Linux is already on the latest version.') + logger.print() + + if len(failed_dbs) > 0: + return 1 + + return 0 diff --git a/src/downloader/offline_importer.py b/src/downloader/offline_importer.py new file mode 100644 index 0000000..c7ab6e4 --- /dev/null +++ b/src/downloader/offline_importer.py @@ -0,0 +1,73 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +from .config import AllowDelete + + +class OfflineImporter: + def __init__(self, config, file_service, logger): + self._config = config + self._file_service = file_service + self._logger = logger + self._dbs = [] + + def add_db(self, db, store): + self._dbs.append((db, store)) + + def apply_offline_databases(self): + for db, store in self._dbs: + for db_file in db['db_files']: + self._update_store_from_offline_db(db['db_id'], db_file, store) + + def _update_store_from_offline_db(self, store_id, db_file, store): + if not self._file_service.is_file(db_file): + return + + hash_db_file = self._file_service.hash(db_file) + if hash_db_file in store['offline_databases_imported']: + self._remove_db_file(db_file) + return + + db = self._file_service.load_db_from_file(db_file) + + if store_id != db['db_id']: + self._logger.print('WARNING! Stored id "%s", doesn\'t match Offline database id "%s" at %s' % ( + store_id, db['db_id'], db_file)) + self._logger.print('Ignoring the offline database.') + return + + self._logger.print('Importing %s into the local store.' % db_file) + + for file_path in db['files']: + if self._file_service.is_file(file_path) and \ + (db['files'][file_path]['hash'] == 'ignore' or self._file_service.hash(file_path) == db['files'][file_path]['hash']) and \ + file_path not in store['files']: + store['files'][file_path] = db['files'][file_path] + + self._logger.print('+', end='', flush=True) + + if len(db['files']) > 0: + self._logger.print() + self._logger.print() + + store['offline_databases_imported'].append(hash_db_file) + self._remove_db_file(db_file) + + def _remove_db_file(self, db_file): + if self._config['allow_delete'] == AllowDelete.ALL: + self._file_service.unlink(db_file) diff --git a/src/downloader/online_importer.py b/src/downloader/online_importer.py new file mode 100644 index 0000000..9f9274f --- /dev/null +++ b/src/downloader/online_importer.py @@ -0,0 +1,121 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +class OnlineImporter: + def __init__(self, config, file_service, downloader_factory, logger): + self._config = config + self._file_service = file_service + self._downloader_factory = downloader_factory + self._logger = logger + self._dbs = [] + self._files_that_failed = [] + self._correctly_installed_files = [] + self._processed_files = {} + self._needs_reboot = False + + def add_db(self, db, store): + self._dbs.append((db, store)) + + def download_dbs_contents(self, full_resync): + for db, store in self._dbs: + self._process_db_contents(db, store, full_resync) + + def _process_db_contents(self, db, store, full_resync): + self._print_db_header(db) + + self._create_folders(db['folders']) + + first_run = len(store['files']) == 0 + self._remove_missing_files(store['files'], db['files']) + + downloader = self._downloader_factory(self._config) + + for file_path in db['files']: + if not full_resync and file_path in store['files'] and \ + store['files'][file_path]['hash'] == db['files'][file_path]['hash'] and \ + self._should_not_download_again(file_path): + continue + + if file_path in self._processed_files: + self._logger.print('DUPLICATED: %s' % file_path) + self._logger.print('Already been processed by database: %s' % self._processed_files[file_path]) + continue + + file_description = db['files'][file_path] + if 'overwrite' in file_description and not file_description['overwrite'] and self._file_service.is_file(file_path): + self._logger.print('%s is already present, and is marked to not be overwritten.' % file_path) + self._logger.print('Delete the file first if you wish to update it.') + continue + + downloader.queue_file(file_description, file_path) + self._processed_files[file_path] = db['db_id'] + + downloader.download_files(first_run) + + for file_path in downloader.errors(): + if file_path in store['files']: + store['files'].pop(file_path) + + self._files_that_failed.extend(downloader.errors()) + + for path in downloader.correctly_downloaded_files(): + store['files'][path] = db['files'][path] + + self._correctly_installed_files.extend(downloader.correctly_downloaded_files()) + + self._needs_reboot = self._needs_reboot or downloader.needs_reboot() + + def _print_db_header(self, db): + if 'header' in db: + for line in db['header']: + self._logger.print(line, end='') + else: + self._logger.print() + self._logger.print('################################################################################') + self._logger.print('SECTION: %s' % db['db_id']) + self._logger.print('################################################################################') + self._logger.print() + + def _should_not_download_again(self, file_path): + if not self._config['check_manually_deleted_files']: + return True + + return self._file_service.is_file(file_path) + + def _create_folders(self, folder_list): + for folder in folder_list: + self._file_service.makedirs(folder) + + def _remove_missing_files(self, store_files, db_files): + files_to_delete = [f for f in store_files if f not in db_files] + + for file_path in files_to_delete: + self._file_service.unlink(file_path) + store_files.pop(file_path) + + if len(files_to_delete) > 0: + self._logger.print() + + def files_that_failed(self): + return self._files_that_failed + + def correctly_installed_files(self): + return self._correctly_installed_files + + def needs_reboot(self): + return self._needs_reboot diff --git a/src/downloader/other.py b/src/downloader/other.py new file mode 100644 index 0000000..1b9c9ab --- /dev/null +++ b/src/downloader/other.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import subprocess +from pathlib import Path + + +def empty_store(): + return { + 'folders': [], + 'files': {}, + 'offline_databases_imported': [] + } + + +def format_files_message(file_list): + any_mra_files = [file for file in file_list if file[-4:].lower() == '.mra'] + + rbfs = [file for file in file_list if file[-4:].lower() == '.rbf' or file == 'MiSTer'] + mras = [file for file in any_mra_files if '/_alternatives/' not in file.lower()] + alts = [file for file in any_mra_files if '/_alternatives/' in file.lower()] + urls = [file for file in file_list if file[0:4].lower() == 'http'] + + printable = [Path(file).name for file in (rbfs + mras)] + urls + if len(alts) > 0: + printable.append('MRA Alternatives') + + there_are_other_files = False + if len(printable) == 0: + printable = file_list + else: + there_are_other_files = len(file_list) > (len(rbfs) + len(mras) + len(alts) + len(urls)) + + message = ', '.join(printable) + if there_are_other_files: + message = '%s + other files.' % message + + return 'none.' if message == '' else message + + +def run_successfully(command, logger): + result = subprocess.run(command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + + stdout = result.stdout.decode() + stderr = result.stderr.decode() + if stdout.strip(): + logger.print(stdout) + + if stderr.strip(): + logger.print(stderr) + + if result.returncode != 0: + raise Exception("subprocess.run %s Return Code was '%d'" % (command, result.returncode)) + + +def run_stdout(command): + result = subprocess.run(command, shell=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE) + + if result.returncode != 0: + raise Exception("subprocess.run %s Return Code was '%d'" % (command, result.returncode)) + + return result.stdout.decode() diff --git a/src/downloader/reboot_calculator.py b/src/downloader/reboot_calculator.py new file mode 100644 index 0000000..43b347c --- /dev/null +++ b/src/downloader/reboot_calculator.py @@ -0,0 +1,57 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +from .config import AllowReboot + + +class RebootCalculator: + def __init__(self, config, logger, file_service): + self._config = config + self._logger = logger + self._file_service = file_service + + def calc_needs_reboot(self, linux_needs_reboot, importer_needs_reboot): + + will_reboot = False + should_reboot = False + + if self._config['allow_reboot'] == AllowReboot.NEVER: + should_reboot = linux_needs_reboot or importer_needs_reboot + elif self._config['allow_reboot'] == AllowReboot.ONLY_AFTER_LINUX_UPDATE: + will_reboot = linux_needs_reboot + should_reboot = importer_needs_reboot + elif self._config['allow_reboot'] == AllowReboot.ALWAYS: + will_reboot = linux_needs_reboot or importer_needs_reboot + else: + raise Exception('AllowReboot.%s not recognized' % AllowReboot.name) + + if will_reboot: + return True + + if should_reboot: + self._file_service.touch(mister_downloader_needs_reboot_file) + if linux_needs_reboot: + self._logger.print('Linux has been updated! It is recommended to reboot your system now.') + else: + self._logger.print('Reboot MiSTer to apply some changes.') + self._logger.print() + + return False + + +mister_downloader_needs_reboot_file = '/tmp/MiSTer_downloader_needs_reboot' diff --git a/src/test/__init__.py b/src/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/test/fake_curl_downloader.py b/src/test/fake_curl_downloader.py new file mode 100644 index 0000000..76bd740 --- /dev/null +++ b/src/test/fake_curl_downloader.py @@ -0,0 +1,71 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer +from downloader.curl_downloader import CurlCommonDownloader as ProductionCurlCommonDownloader +from test.fake_file_service import FileService +from test.fake_logger import NoLogger +from test.objects import file_MiSTer, file_MiSTer_new + + +def downloader_with_errors(errors): + downloader = CurlDownloader() + for file_path in errors: + downloader.test_data.errors_at(file_path) + return downloader + + +class TestDataCurlDownloader: + def __init__(self, problematic_files): + self._problematic_files = problematic_files + + def errors_at(self, file, tries=None): + self._problematic_files[file] = tries if tries is not None else 99 + return self + + +class CurlDownloader(ProductionCurlCommonDownloader): + def __init__(self, config=None, file_service=None, problematic_files=None): + config = config if config is not None else {'curl_ssl': '', 'downloader_retries': 3} + self.file_service = FileService() if file_service is None else file_service + super().__init__(config, self.file_service, NoLogger()) + self._run_files = [] + self._problematic_files = dict() if problematic_files is None else problematic_files + + @property + def test_data(self): + return TestDataCurlDownloader(self._problematic_files) + + def _run(self, description, target_path, file): + self._run_files.append(file) + + if file in self._problematic_files: + self._problematic_files[file] -= 1 + + if file not in self._problematic_files or self._problematic_files[file] <= 0: + self._file_service.test_data.with_file(file if file != file_MiSTer else file_MiSTer_new, description) + self._http_oks.append(file) + else: + self._errors.append(file) + + def _command(self, target_path, url): + return target_path + + def _wait(self): + pass + + def run_files(self): + return self._run_files \ No newline at end of file diff --git a/src/test/fake_db_gateway.py b/src/test/fake_db_gateway.py new file mode 100644 index 0000000..b0f1fc2 --- /dev/null +++ b/src/test/fake_db_gateway.py @@ -0,0 +1,27 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +class DbGateway: + def __init__(self, dbs=None): + self._dbs = dbs + + def fetch(self, db_uri): + if self._dbs is None: + return None + + return self._dbs[db_uri] \ No newline at end of file diff --git a/src/test/fake_file_service.py b/src/test/fake_file_service.py new file mode 100644 index 0000000..6f45434 --- /dev/null +++ b/src/test/fake_file_service.py @@ -0,0 +1,115 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer +import test.objects + + +class TestDataFileService: + def __init__(self, files): + self._files = files + + def with_file(self, file, description): + self._files.add(file, description) + return self + + def with_file_a(self, description=None): + self._files.add(test.objects.file_a, description if description is not None else test.objects.file_a_descr()) + return self + + def with_test_json_zip(self, description=None): + self._files.add(test.objects.file_test_json_zip, description if description is not None else test.objects.file_test_json_zip_descr()) + return self + + +class FileService: + def __init__(self): + self._files = CaseInsensitiveDict() + + @property + def test_data(self): + return TestDataFileService(self._files) + + def add_system_path(self, path): + pass + + def is_file(self, path): + return self._files.has(path) + + def read_file_contents(self, path): + if not self._files.has(path): + return 'unknown' + return self._files.get(path)['content'] + + def write_file_contents(self, path, content): + if not self._files.has(path): + self._files.add(path, {}) + self._files.get(path)['content'] = content + + def touch(self, path): + self._files.add(path, {'hash': path}) + + def move(self, source, target): + self._files.add(target, {'hash': target}) + self._files.pop(source) + + def copy(self, source, target): + self._files.add(target, self._files.get(source)) + + def hash(self, path): + return self._files.get(path)['hash'] + + def makedirs(self, path): + pass + + def makedirs_parent(self, path): + pass + + def curl_target_path(self, path): + return path + + def unlink(self, path): + if self._files.has(path): + self._files.pop(path) + + def clean_expression(self, expr): + if expr[-1:] == '*': + pass + else: + self.unlink(expr) + + def save_json_on_zip(self, db, path): + pass + + def load_db_from_file(self, path): + return self._files.get(path)['unzipped_json'] + + +class CaseInsensitiveDict: + def __init__(self): + self._dict = dict() + + def add(self, key, value): + self._dict[key.lower()] = value + + def get(self, key): + return self._dict[key.lower()] + + def has(self, key): + return key.lower() in self._dict + + def pop(self, key): + self._dict.pop(key.lower()) diff --git a/src/test/fake_logger.py b/src/test/fake_logger.py new file mode 100644 index 0000000..896e931 --- /dev/null +++ b/src/test/fake_logger.py @@ -0,0 +1,32 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer +import sys + + +class NoLogger: + def print(self, *args, sep='', end='\n', file=sys.stdout, flush=False): + pass + + def debug(self, *args, sep='', end='\n', file=sys.stdout, flush=False): + pass + + def set_local_repository(self, local_repository): + pass + + def close_logfile(self): + pass diff --git a/src/test/fakes.py b/src/test/fakes.py new file mode 100644 index 0000000..e65c934 --- /dev/null +++ b/src/test/fakes.py @@ -0,0 +1,88 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer +from pathlib import Path + +from downloader.config import default_config +from downloader.online_importer import OnlineImporter as ProductionOnlineImporter +from downloader.offline_importer import OfflineImporter as ProductionOfflineImporter +from downloader.config import ConfigReader as ProductionConfigReader +from downloader.reboot_calculator import RebootCalculator as ProductionRebootCalculator +from downloader.local_repository import LocalRepository as ProductionLocalRepository +from downloader.linux_updater import LinuxUpdater as ProductionLinuxUpdater +from test.fake_file_service import FileService +from test.fake_curl_downloader import CurlDownloader, TestDataCurlDownloader +from test.fake_logger import NoLogger +from test.objects import default_env + + +class ConfigReader(ProductionConfigReader): + def __init__(self): + super().__init__(NoLogger(), default_env()) + + +class OnlineImporter(ProductionOnlineImporter): + def __init__(self, downloader=None, config=None, file_service=None): + self.file_service = FileService() if file_service is None else file_service + self._problematic_files = dict() + config = default_config() if config is None else config + super().__init__( + config, + self.file_service, + lambda c: CurlDownloader(c, self.file_service, self._problematic_files) if downloader is None else downloader, + NoLogger()) + + @property + def downloader_test_data(self): + return TestDataCurlDownloader(self._problematic_files) + + +class OfflineImporter(ProductionOfflineImporter): + def __init__(self, config=None, file_service=None): + self.file_service = FileService() if file_service is None else file_service + super().__init__( + default_config() if config is None else config, + self.file_service, + NoLogger()) + + +class RebootCalculator(ProductionRebootCalculator): + def __init__(self, config=None, file_service=None): + self.file_service = FileService() if file_service is None else file_service + super().__init__(default_config() if config is None else config, NoLogger(), self.file_service) + + +class LocalRepository(ProductionLocalRepository): + def __init__(self, config=None, file_service=None): + self.file_service = FileService() if file_service is None else file_service + super().__init__(self._config() if config is None else config, NoLogger(), self.file_service) + + def _config(self): + config = default_config() + config['config_path'] = Path('') + return config + + +class LinuxUpdater(ProductionLinuxUpdater): + def __init__(self, file_service=None, downloader=None): + self.file_service = FileService() if file_service is None else file_service + self.downloader = CurlDownloader(default_config(), self.file_service) if downloader is None else downloader + super().__init__(self.file_service, self.downloader, NoLogger()) + + def _run_subprocesses(self, linux, linux_path): + self.file_service.write_file_contents('/MiSTer.version', linux['version']) + self.file_service.touch('/tmp/downloader_needs_reboot_after_linux_update') diff --git a/src/test/integration/__init__.py b/src/test/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/test/integration/fixtures/custom_mister.ini b/src/test/integration/fixtures/custom_mister.ini new file mode 100644 index 0000000..15995fc --- /dev/null +++ b/src/test/integration/fixtures/custom_mister.ini @@ -0,0 +1,8 @@ +[mister] +update_linux = false +parallel_update = false +check_manually_deleted_files = 1 +allow_reboot = 0 +allow_delete = 2 +base_path = '/media/usb0/' +base_system_path = '/media/cifs/' \ No newline at end of file diff --git a/src/test/integration/fixtures/custom_mister_dbs.ini b/src/test/integration/fixtures/custom_mister_dbs.ini new file mode 100644 index 0000000..39813a8 --- /dev/null +++ b/src/test/integration/fixtures/custom_mister_dbs.ini @@ -0,0 +1,14 @@ +[mister] +update_linux = 'False' +parallel_update = "true" +check_manually_deleted_files = 0 +allow_reboot = "2" +allow_delete = '0' +base_path = '/media/usb1/' +base_system_path = '/media/usb2/' + +[Single] +db_url = "https://single.com" + +[Double] +db_url = "https://double.com" \ No newline at end of file diff --git a/src/test/integration/fixtures/db_plus_distrib_empty_section_1.ini b/src/test/integration/fixtures/db_plus_distrib_empty_section_1.ini new file mode 100644 index 0000000..895a9c5 --- /dev/null +++ b/src/test/integration/fixtures/db_plus_distrib_empty_section_1.ini @@ -0,0 +1,3 @@ +[distribution_mister] +[One] +db_url = "https://one.com" \ No newline at end of file diff --git a/src/test/integration/fixtures/db_plus_distrib_empty_section_2.ini b/src/test/integration/fixtures/db_plus_distrib_empty_section_2.ini new file mode 100644 index 0000000..0404fc0 --- /dev/null +++ b/src/test/integration/fixtures/db_plus_distrib_empty_section_2.ini @@ -0,0 +1,3 @@ +[One] +db_url = "https://one.com" +[distribution_mister] diff --git a/src/test/integration/fixtures/db_plus_random_empty_section.ini b/src/test/integration/fixtures/db_plus_random_empty_section.ini new file mode 100644 index 0000000..af72218 --- /dev/null +++ b/src/test/integration/fixtures/db_plus_random_empty_section.ini @@ -0,0 +1,3 @@ +[Random] +[One] +db_url = "https://one.com" \ No newline at end of file diff --git a/src/test/integration/fixtures/double_db.ini b/src/test/integration/fixtures/double_db.ini new file mode 100644 index 0000000..b330cd3 --- /dev/null +++ b/src/test/integration/fixtures/double_db.ini @@ -0,0 +1,5 @@ +[Single] +db_url = "https://single.com" + +[Double] +db_url = "https://double.com" \ No newline at end of file diff --git a/src/test/integration/fixtures/single_db.ini b/src/test/integration/fixtures/single_db.ini new file mode 100644 index 0000000..6e3938c --- /dev/null +++ b/src/test/integration/fixtures/single_db.ini @@ -0,0 +1,2 @@ +[Single] +db_url = "https://single.com" \ No newline at end of file diff --git a/src/test/integration/test_config_reader.py b/src/test/integration/test_config_reader.py new file mode 100644 index 0000000..8c6b740 --- /dev/null +++ b/src/test/integration/test_config_reader.py @@ -0,0 +1,136 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +from downloader.config import AllowDelete, AllowReboot +from test.objects import not_found_ini +from test.fakes import ConfigReader + + +class TestConfigReader(unittest.TestCase): + def setUp(self): + self.maxDiff = None + + def test_config_reader___when_no_ini___returns_default_values(self): + self.assertConfig(not_found_ini(), { + 'update_linux': True, + 'parallel_update': True, + 'allow_reboot': AllowReboot.ALWAYS, + 'allow_delete': AllowDelete.ALL, + 'check_manually_deleted_files': True, + 'base_path': '/media/fat/', + 'base_system_path': '/media/fat/', + 'downloader_size_mb_limit': 100, + 'downloader_process_limit': 300, + 'downloader_timeout': 300, + 'downloader_retries': 3, + 'databases': [{ + 'db_url': 'https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/db.json.zip', + 'section': 'distribution_mister', + }] + }) + + def test_databases___with_single_db_ini___returns_single_db_only(self): + self.assertEqual(self.databases("test/integration/fixtures/single_db.ini"), [{ + 'db_url': 'https://single.com', + 'section': 'Single', + }, ]) + + def test_databases___with_dobule_db_ini___returns_dobule_db_only(self): + self.assertEqual(self.databases("test/integration/fixtures/double_db.ini"), [{ + 'db_url': 'https://single.com', + 'section': 'Single', + }, { + 'db_url': 'https://double.com', + 'section': 'Double', + }]) + + def test_config_reader___with_custom_mister_ini___returns_custom_fields(self): + self.assertConfig("test/integration/fixtures/custom_mister.ini", { + 'update_linux': False, + 'parallel_update': False, + 'check_manually_deleted_files': True, + 'allow_reboot': AllowReboot.NEVER, + 'allow_delete': AllowDelete.OLD_RBF, + 'base_path': '/media/usb0/', + 'base_system_path': '/media/cifs/', + 'databases': [{ + 'db_url': 'https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/db.json.zip', + 'section': 'distribution_mister', + }], + }) + + def test_config_reader___with_custom_mister_dbs_ini___returns_custom_fields_and_dbs(self): + self.assertConfig("test/integration/fixtures/custom_mister_dbs.ini", { + 'update_linux': False, + 'parallel_update': True, + 'check_manually_deleted_files': False, + 'allow_reboot': AllowReboot.ONLY_AFTER_LINUX_UPDATE, + 'allow_delete': AllowDelete.NONE, + 'base_path': '/media/usb1/', + 'base_system_path': '/media/usb2/', + 'databases': [{ + 'db_url': 'https://single.com', + 'section': 'Single', + }, { + 'db_url': 'https://double.com', + 'section': 'Double', + }], + }) + + def test_config_reader___with_db_and_distrib_empty_section_1___returns_one_db_and_defult_distrib(self): + self.assertEqual(self.databases("test/integration/fixtures/db_plus_distrib_empty_section_1.ini"), [{ + 'db_url': 'https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/db.json.zip', + 'section': 'distribution_mister', + }, { + 'db_url': 'https://one.com', + 'section': 'One', + }, ]) + + def test_config_reader___with_db_and_distrib_empty_section_2___returns_one_db_and_defult_distrib(self): + self.assertEqual(self.databases("test/integration/fixtures/db_plus_distrib_empty_section_2.ini"), [{ + 'db_url': 'https://one.com', + 'section': 'One', + }, { + 'db_url': 'https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/db.json.zip', + 'section': 'distribution_mister', + }, ]) + + def test_config_reader___with_db_and_random_empty_section_2___raises_exception(self): + self.assertRaises(Exception, lambda: self.databases("test/integration/fixtures/db_plus_random_empty_section.ini")) + + def assertConfig(self, path, config_vars): + actual = ConfigReader().read_config(path) + + expected = {} + + vars_count = 0 + + for key in actual: + if key in config_vars: + vars_count = vars_count + 1 + expected[key] = config_vars[key] + else: + expected[key] = actual[key] + + self.assertGreater(vars_count, 0, "No valid config_vars have been provided.") + self.assertEqual(len(config_vars), vars_count, "Some config_vars were misspelled.") + self.assertEqual(actual, expected) + + def databases(self, path): + return ConfigReader().read_config(path)['databases'] diff --git a/src/test/integration/test_file_service.py b/src/test/integration/test_file_service.py new file mode 100644 index 0000000..e82b9ff --- /dev/null +++ b/src/test/integration/test_file_service.py @@ -0,0 +1,142 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +from downloader.file_service import FileService +from downloader.config import AllowDelete, default_config +from test.fakes import NoLogger +from pathlib import Path +import shutil +import os + +not_created_file = '/tmp/opened_file' +empty_file = '/tmp/empty_file' +mister_file = 'MiSTer' +rbf_file = 'MyCore.RBF' +delme_dir = 'file_service_integration_delme' + + +class TestFileService(unittest.TestCase): + + def setUp(self) -> None: + Path(empty_file).touch() + + def tearDown(self) -> None: + self.unlink(not_created_file) + self.unlink(empty_file) + shutil.rmtree(delme_dir, ignore_errors=True) + + def test_is_file___on_nothing__returns_false(self): + self.assertFalse(self.sut().is_file('this_will_never_be_a_file.sh')) + + def test_is_file___on_read_file_contents__returns_content(self): + self.sut().touch(not_created_file) + actual = self.sut().read_file_contents(not_created_file) + self.assertEqual(actual, '') + + def test_is_file___on_touched_file__returns_true(self): + self.sut().touch(not_created_file) + self.assertTrue(self.sut().is_file(not_created_file)) + + def test_hash___on_empty_file__returns_same_string_always(self): + self.assertEqual(self.sut().hash(empty_file), "d41d8cd98f00b204e9800998ecf8427e") + + def test_hash___on_bigger_file__returns_different_string(self): + self.assertNotEqual(self.sut({'base_path': '..'}).hash('downloader.sh'), "d41d8cd98f00b204e9800998ecf8427e") + + def test_move___on_existing_file__works_fine(self): + self.sut().move(empty_file, not_created_file) + self.assertTrue(self.sut().is_file(not_created_file)) + self.assertFalse(self.sut().is_file(empty_file)) + + def test_copy___on_existing_file__works_fine(self): + self.sut().copy(empty_file, not_created_file) + self.assertTrue(self.sut().is_file(not_created_file)) + self.assertTrue(self.sut().is_file(empty_file)) + + def test_unlink_mister___when_allow_delete_only_rbf___deletes_it(self): + sut = self.sut({ 'allow_delete': AllowDelete.OLD_RBF, 'base_path': delme_dir, 'base_system_path': delme_dir }) + + sut.makedirs_parent(mister_file) + sut.touch(mister_file) + self.assertTrue(sut.is_file(mister_file)) + + sut.unlink(mister_file) + self.assertFalse(sut.is_file(mister_file)) + + def test_unlink_mister_file___when_allow_delete_none___doesnt_delete_it(self): + sut = self.sut({ 'allow_delete': AllowDelete.NONE, 'base_path': delme_dir, 'base_system_path': delme_dir }) + + sut.makedirs_parent(mister_file) + sut.touch(mister_file) + self.assertTrue(sut.is_file(mister_file)) + + sut.unlink(mister_file) + self.assertTrue(sut.is_file(mister_file)) + + def test_unlink_rbf_file___when_allow_delete_only_rbf___deletes_it(self): + sut = self.sut({ 'allow_delete': AllowDelete.OLD_RBF, 'base_path': delme_dir, 'base_system_path': delme_dir }) + + sut.makedirs_parent(rbf_file) + sut.touch(rbf_file) + self.assertTrue(sut.is_file(rbf_file)) + + sut.unlink(rbf_file) + self.assertFalse(sut.is_file(rbf_file)) + + def test_unlink_something_else___when_allow_delete_only_rbf___doesnt_delete_it(self): + sut = self.sut({ 'allow_delete': AllowDelete.OLD_RBF, 'base_path': delme_dir, 'base_system_path': delme_dir }) + + sut.makedirs_parent(empty_file) + sut.touch(empty_file) + self.assertTrue(sut.is_file(empty_file)) + + sut.unlink(empty_file) + self.assertTrue(sut.is_file(empty_file)) + + def test_unlink_anything___with_default_config___deletes_it(self): + self.sut().touch(empty_file) + self.assertTrue(self.sut().is_file(empty_file)) + + self.sut().unlink(empty_file) + self.assertFalse(self.sut().is_file(empty_file)) + + def test_curl_path_x___after_add_system_path_x_with_system_path_b___returns_b_plus_x(self): + sut = self.sut({'base_path': 'a', 'base_system_path': 'b'}) + sut.add_system_path('x') + self.assertEqual(sut.curl_target_path('x'), 'b/x') + + def test_curl_path_x___with_system_path_a___returns_a_plus_x(self): + sut = self.sut({'base_path': 'a', 'base_system_path': 'b'}) + self.assertEqual(sut.curl_target_path('x'), 'a/x') + + def test_curl_path_temp_x___always___returns_temp_plus_x(self): + self.assertEqual(self.sut().curl_target_path('/tmp/x'), '/tmp/x') + + def test_makedirs___on_missing_folder___creates_it(self): + self.sut({'base_path': delme_dir}).makedirs('foo') + self.assertTrue(os.path.isdir(delme_dir + '/foo')) + + def sut(self, config=None): + return FileService(default_config() if config is None else config, NoLogger()) + + def unlink(self, file): + try: + Path(file).unlink() + except: + pass \ No newline at end of file diff --git a/src/test/objects.py b/src/test/objects.py new file mode 100644 index 0000000..ec66da9 --- /dev/null +++ b/src/test/objects.py @@ -0,0 +1,146 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +from pathlib import Path +import copy + +file_test_json_zip = 'test.json.zip' +file_a = 'A' +file_boot_rom = 'boot.rom' +file_menu_rbf = 'menu.rbf' +hash_menu_rbf = 'menu.rbf' +file_MiSTer = 'MiSTer' +hash_MiSTer = 'MiSTer.new' +file_MiSTer_new = 'MiSTer.new' +folder_a = 'a' +db_test = 'test' +file_one = 'one' +hash_one = 'one' + + +def file_test_json_zip_descr(): + return {'hash': file_test_json_zip, 'unzipped_json': db_test_with_file_a_descr()} + + +def db_empty_descr(): + return { + 'db_id': db_test, + 'db_files': [''], + 'files': {}, + 'folders': [] + } + + +def file_mister_descr(): + return { + "delete": [], + "hash": hash_MiSTer, + "size": 2915040, + "url": "https://MiSTer", + "reboot": True, + "path": "system" + } + + +def file_a_descr(): + return { + "delete": [], + "hash": file_a, + "size": 2915040, + "url": "https://one.rbf" + } + + +def boot_rom_descr(): + return { + "delete": [], + "hash": file_boot_rom, + "size": 29315040, + "url": "https://boot.rom", + "overwrite": False + } + + +def overwrite_file(descr, overwrite): + result = copy.deepcopy(descr) + result['overwrite'] = overwrite + return result + + +def file_a_updated_descr(): + return { + "delete": [], + "hash": "B946e696994573394343edb74c54180c", + "size": 2915040, + "url": "https://one.rbf" + } + + +def db_test_with_file(name_file, file): + return { + 'db_id': db_test, + 'db_files': [file_test_json_zip], + 'files': { + name_file: file + }, + 'folders': [] + } + + +def db_with_file(db_id, name_file, file): + return { + 'db_id': db_id, + 'db_files': [db_id + '.json.zip'], + 'files': { + name_file: file + }, + 'folders': [] + } + + +def db_test_with_file_a_descr(): + return { + 'db_id': db_test, + 'db_files': [file_test_json_zip], + 'files': { + file_a: file_a_descr() + }, + 'folders': [folder_a] + } + + +def not_found_sh(): + return _not_file('not_found.sh') + + +def not_found_ini(): + return _not_file('not_found.ini') + + +def _not_file(file): + unittest.TestCase().assertFalse(Path(file).is_file()) + return file + + +def default_env(): + return { + 'DEFAULT_DB_URL': 'https://raw.githubusercontent.com/MiSTer-devel/Distribution_MiSTer/main/db.json.zip', + 'DEFAULT_DB_ID': 'distribution_mister', + 'ALLOW_REBOOT': None + } diff --git a/src/test/system/fixtures/full_install/parallel.ini b/src/test/system/fixtures/full_install/parallel.ini new file mode 100644 index 0000000..8b3c0bf --- /dev/null +++ b/src/test/system/fixtures/full_install/parallel.ini @@ -0,0 +1,6 @@ +[mister] +update_linux = false +parallel_update = true +allow_reboot = 2 +base_path = '/tmp/delme_parallel' +base_system_path = '/tmp/delme_parallel' \ No newline at end of file diff --git a/src/test/system/fixtures/full_install/serial.ini b/src/test/system/fixtures/full_install/serial.ini new file mode 100644 index 0000000..2b0aec9 --- /dev/null +++ b/src/test/system/fixtures/full_install/serial.ini @@ -0,0 +1,6 @@ +[mister] +update_linux = false +parallel_update = false +allow_reboot = 2 +base_path = '/tmp/delme_serial' +base_system_path = '/tmp/delme_serial' \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/files/bar.txt b/src/test/system/fixtures/sandboxed_install/files/bar.txt new file mode 100644 index 0000000..c903be1 --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/files/bar.txt @@ -0,0 +1,11 @@ + + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur iaculis pellentesque tellus in condimentum. Pellentesque volutpat convallis lorem at aliquet. Sed nec odio at ligula tempor hendrerit vitae in est. Maecenas molestie condimentum magna, aliquet bibendum sem auctor consectetur. Donec mattis, eros sed vehicula egestas, risus turpis auctor urna, vel dictum justo ex eget erat. Etiam dictum lobortis arcu, ut vestibulum ante. In a dolor sit amet risus porta hendrerit. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mauris odio, consectetur sit amet tortor sit amet, rhoncus pharetra orci. In id lectus nec tellus porttitor placerat quis vel nisi. Nam maximus neque turpis, a tincidunt metus varius vel. Aliquam congue mauris ut interdum vestibulum. Duis finibus nisl ac ultricies ullamcorper. Duis imperdiet turpis ac eros venenatis, a mollis arcu sodales. + +Nam auctor varius neque id lobortis. Pellentesque viverra malesuada viverra. Sed accumsan dui et convallis rhoncus. Aenean pellentesque pretium eros sit amet iaculis. Maecenas et massa convallis, pulvinar tortor in, sollicitudin metus. Quisque mauris nulla, eleifend eget commodo sit amet, gravida vestibulum urna. Aenean et odio eu turpis sagittis euismod. Pellentesque mollis sem turpis, a consequat erat aliquet a. + +Aenean sit amet purus elit. Donec finibus lorem quis consectetur elementum. Aliquam in cursus dolor. Fusce id volutpat quam, vitae semper enim. Duis ullamcorper leo eu ullamcorper pellentesque. Cras ipsum tellus, efficitur eget commodo ac, iaculis pharetra massa. Donec sed dolor lectus. Cras a velit quis justo placerat porta. Curabitur et nibh vel turpis porta elementum in non quam. Vivamus euismod malesuada lacus, in sollicitudin est iaculis non. Ut tincidunt nulla erat, vitae sagittis tellus vulputate eget. Vivamus nec nisl at mi bibendum hendrerit. Etiam id nunc varius, efficitur arcu non, pulvinar libero. Fusce erat ex, volutpat fringilla nisl id, imperdiet tristique ligula. Ut neque sapien, feugiat non magna ut, luctus bibendum nunc. + +In hac habitasse platea dictumst. Integer vel libero eu ex efficitur interdum. Curabitur tincidunt pretium mauris, ac convallis ligula condimentum a. Cras nibh velit, varius facilisis leo non, sagittis iaculis nisi. Donec ut placerat quam, ac blandit felis. Sed ac tellus lobortis, ultrices dolor ut, luctus turpis. Vivamus vulputate ligula in nisi molestie, sit amet iaculis enim aliquet. Aenean vehicula, sem sit amet commodo imperdiet, tellus risus scelerisque libero, nec pretium nulla ante in tortor. Aliquam imperdiet urna a pharetra eleifend. Integer eget rutrum purus, quis accumsan diam. Vivamus porttitor urna nec ex cursus eleifend. Vestibulum quis sem ac justo laoreet imperdiet eu ac enim. Aliquam sed ipsum vel est venenatis vulputate eget ut sapien. Fusce id venenatis lectus. Curabitur aliquet, turpis vel semper placerat, leo arcu facilisis tellus, vel aliquet dolor dolor nec orci. Nulla condimentum velit a nisl feugiat malesuada. + +Fusce tempor nibh non augue hendrerit vehicula. Mauris eleifend metus sollicitudin quam tempus vulputate. Nam dignissim pharetra augue ac pellentesque. Phasellus ornare sit amet nisi eget laoreet. Nunc finibus molestie quam lacinia blandit. Ut gravida ligula ut tempor vestibulum. Donec vulputate turpis ligula, non dictum est congue et. Praesent mauris nibh, fringilla porttitor bibendum ut, sagittis ac nunc. Mauris eu orci sagittis, tempor mauris eget, suscipit magna. Aenean ut ex tortor. Nullam consequat, leo in interdum tincidunt, nunc felis maximus nibh, id faucibus mauris nisi id elit. Nunc auctor lorem turpis, a commodo eros luctus non. Nam ac tristique risus. Nunc pellentesque leo lectus, et mollis magna dictum sed. Suspendisse blandit quam augue, nec luctus lectus cursus a. \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/files/bar_updated.txt b/src/test/system/fixtures/sandboxed_install/files/bar_updated.txt new file mode 100644 index 0000000..805a8f7 --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/files/bar_updated.txt @@ -0,0 +1,19 @@ + + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur iaculis pellentesque tellus in condimentum. Pellentesque volutpat convallis lorem at aliquet. Sed nec odio at ligula tempor hendrerit vitae in est. Maecenas molestie condimentum magna, aliquet bibendum sem auctor consectetur. Donec mattis, eros sed vehicula egestas, risus turpis auctor urna, vel dictum justo ex eget erat. Etiam dictum lobortis arcu, ut vestibulum ante. In a dolor sit amet risus porta hendrerit. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mauris odio, consectetur sit amet tortor sit amet, rhoncus pharetra orci. In id lectus nec tellus porttitor placerat quis vel nisi. Nam maximus neque turpis, a tincidunt metus varius vel. Aliquam congue mauris ut interdum vestibulum. Duis finibus nisl ac ultricies ullamcorper. Duis imperdiet turpis ac eros venenatis, a mollis arcu sodales. + +Nam auctor varius neque id lobortis. Pellentesque viverra malesuada viverra. Sed accumsan dui et convallis rhoncus. Aenean pellentesque pretium eros sit amet iaculis. Maecenas et massa convallis, pulvinar tortor in, sollicitudin metus. Quisque mauris nulla, eleifend eget commodo sit amet, gravida vestibulum urna. Aenean et odio eu turpis sagittis euismod. Pellentesque mollis sem turpis, a consequat erat aliquet a. + +Aenean sit amet purus elit. Donec finibus lorem quis consectetur elementum. Aliquam in cursus dolor. Fusce id volutpat quam, vitae semper enim. Duis ullamcorper leo eu ullamcorper pellentesque. Cras ipsum tellus, efficitur eget commodo ac, iaculis pharetra massa. Donec sed dolor lectus. Cras a velit quis justo placerat porta. Curabitur et nibh vel turpis porta elementum in non quam. Vivamus euismod malesuada lacus, in sollicitudin est iaculis non. Ut tincidunt nulla erat, vitae sagittis tellus vulputate eget. Vivamus nec nisl at mi bibendum hendrerit. Etiam id nunc varius, efficitur arcu non, pulvinar libero. Fusce erat ex, volutpat fringilla nisl id, imperdiet tristique ligula. Ut neque sapien, feugiat non magna ut, luctus bibendum nunc. + +In hac habitasse platea dictumst. Integer vel libero eu ex efficitur interdum. Curabitur tincidunt pretium mauris, ac convallis ligula condimentum a. Cras nibh velit, varius facilisis leo non, sagittis iaculis nisi. Donec ut placerat quam, ac blandit felis. Sed ac tellus lobortis, ultrices dolor ut, luctus turpis. Vivamus vulputate ligula in nisi molestie, sit amet iaculis enim aliquet. Aenean vehicula, sem sit amet commodo imperdiet, tellus risus scelerisque libero, nec pretium nulla ante in tortor. Aliquam imperdiet urna a pharetra eleifend. Integer eget rutrum purus, quis accumsan diam. Vivamus porttitor urna nec ex cursus eleifend. Vestibulum quis sem ac justo laoreet imperdiet eu ac enim. Aliquam sed ipsum vel est venenatis vulputate eget ut sapien. Fusce id venenatis lectus. Curabitur aliquet, turpis vel semper placerat, leo arcu facilisis tellus, vel aliquet dolor dolor nec orci. Nulla condimentum velit a nisl feugiat malesuada. + +Fusce tempor nibh non augue hendrerit vehicula. Mauris eleifend metus sollicitudin quam tempus vulputate. Nam dignissim pharetra augue ac pellentesque. Phasellus ornare sit amet nisi eget laoreet. Nunc finibus molestie quam lacinia blandit. Ut gravida ligula ut tempor vestibulum. Donec vulputate turpis ligula, non dictum est congue et. Praesent mauris nibh, fringilla porttitor bibendum ut, sagittis ac nunc. Mauris eu orci sagittis, tempor mauris eget, suscipit magna. Aenean ut ex tortor. Nullam consequat, leo in interdum tincidunt, nunc felis maximus nibh, id faucibus mauris nisi id elit. Nunc auctor lorem turpis, a commodo eros luctus non. Nam ac tristique risus. Nunc pellentesque leo lectus, et mollis magna dictum sed. Suspendisse blandit quam augue, nec luctus lectus cursus a. + + + +Aenean sit amet purus elit. Donec finibus lorem quis consectetur elementum. Aliquam in cursus dolor. Fusce id volutpat quam, vitae semper enim. Duis ullamcorper leo eu ullamcorper pellentesque. Cras ipsum tellus, efficitur eget commodo ac, iaculis pharetra massa. Donec sed dolor lectus. Cras a velit quis justo placerat porta. Curabitur et nibh vel turpis porta elementum in non quam. Vivamus euismod malesuada lacus, in sollicitudin est iaculis non. Ut tincidunt nulla erat, vitae sagittis tellus vulputate eget. Vivamus nec nisl at mi bibendum hendrerit. Etiam id nunc varius, efficitur arcu non, pulvinar libero. Fusce erat ex, volutpat fringilla nisl id, imperdiet tristique ligula. Ut neque sapien, feugiat non magna ut, luctus bibendum nunc. + +In hac habitasse platea dictumst. Integer vel libero eu ex efficitur interdum. Curabitur tincidunt pretium mauris, ac convallis ligula condimentum a. Cras nibh velit, varius facilisis leo non, sagittis iaculis nisi. Donec ut placerat quam, ac blandit felis. Sed ac tellus lobortis, ultrices dolor ut, luctus turpis. Vivamus vulputate ligula in nisi molestie, sit amet iaculis enim aliquet. Aenean vehicula, sem sit amet commodo imperdiet, tellus risus scelerisque libero, nec pretium nulla ante in tortor. Aliquam imperdiet urna a pharetra eleifend. Integer eget rutrum purus, quis accumsan diam. Vivamus porttitor urna nec ex cursus eleifend. Vestibulum quis sem ac justo laoreet imperdiet eu ac enim. Aliquam sed ipsum vel est venenatis vulputate eget ut sapien. Fusce id venenatis lectus. Curabitur aliquet, turpis vel semper placerat, leo arcu facilisis tellus, vel aliquet dolor dolor nec orci. Nulla condimentum velit a nisl feugiat malesuada. + +Fusce tempor nibh non augue hendrerit vehicula. Mauris eleifend metus sollicitudin quam tempus vulputate. Nam dignissim pharetra augue ac pellentesque. Phasellus ornare sit amet nisi eget laoreet. Nunc finibus molestie quam lacinia blandit. Ut gravida ligula ut tempor vestibulum. Donec vulputate turpis ligula, non dictum est congue et. Praesent mauris nibh, fringilla porttitor bibendum ut, sagittis ac nunc. Mauris eu orci sagittis, tempor mauris eget, suscipit magna. Aenean ut ex tortor. Nullam consequat, leo in interdum tincidunt, nunc felis maximus nibh, id faucibus mauris nisi id elit. Nunc auctor lorem turpis, a commodo eros luctus non. Nam ac tristique risus. Nunc pellentesque leo lectus, et mollis magna dictum sed. Suspendisse blandit quam augue, nec luctus lectus cursus a. \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/files/baz.txt b/src/test/system/fixtures/sandboxed_install/files/baz.txt new file mode 100644 index 0000000..047128d --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/files/baz.txt @@ -0,0 +1,3 @@ +Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem. Ut enim ad minima veniam, quis nostrum exercitationem ullam corporis suscipit laboriosam, nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure reprehenderit qui in ea voluptate velit esse quam nihil molestiae consequatur, vel illum qui dolorem eum fugiat quo voluptas nulla pariatur? At vero eos et accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias excepturi sint occaecati cupiditate non provident, similique sunt in culpa qui officia deserunt mollitia animi, id est laborum et dolorum fuga. Et harum quidem rerum facilis est et expedita distinctio. + +Nam libero tempore, cum soluta nobis est eligendi optio cumque nihil impedit quo minus id quod maxime placeat facere \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/files/foo.txt b/src/test/system/fixtures/sandboxed_install/files/foo.txt new file mode 100644 index 0000000..0ffd59b --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/files/foo.txt @@ -0,0 +1,51 @@ + + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur iaculis pellentesque tellus in condimentum. Pellentesque volutpat convallis lorem at aliquet. Sed nec odio at ligula tempor hendrerit vitae in est. Maecenas molestie condimentum magna, aliquet bibendum sem auctor consectetur. Donec mattis, eros sed vehicula egestas, risus turpis auctor urna, vel dictum justo ex eget erat. Etiam dictum lobortis arcu, ut vestibulum ante. In a dolor sit amet risus porta hendrerit. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mauris odio, consectetur sit amet tortor sit amet, rhoncus pharetra orci. In id lectus nec tellus porttitor placerat quis vel nisi. Nam maximus neque turpis, a tincidunt metus varius vel. Aliquam congue mauris ut interdum vestibulum. Duis finibus nisl ac ultricies ullamcorper. Duis imperdiet turpis ac eros venenatis, a mollis arcu sodales. + +Nam auctor varius neque id lobortis. Pellentesque viverra malesuada viverra. Sed accumsan dui et convallis rhoncus. Aenean pellentesque pretium eros sit amet iaculis. Maecenas et massa convallis, pulvinar tortor in, sollicitudin metus. Quisque mauris nulla, eleifend eget commodo sit amet, gravida vestibulum urna. Aenean et odio eu turpis sagittis euismod. Pellentesque mollis sem turpis, a consequat erat aliquet a. + +Aenean sit amet purus elit. Donec finibus lorem quis consectetur elementum. Aliquam in cursus dolor. Fusce id volutpat quam, vitae semper enim. Duis ullamcorper leo eu ullamcorper pellentesque. Cras ipsum tellus, efficitur eget commodo ac, iaculis pharetra massa. Donec sed dolor lectus. Cras a velit quis justo placerat porta. Curabitur et nibh vel turpis porta elementum in non quam. Vivamus euismod malesuada lacus, in sollicitudin est iaculis non. Ut tincidunt nulla erat, vitae sagittis tellus vulputate eget. Vivamus nec nisl at mi bibendum hendrerit. Etiam id nunc varius, efficitur arcu non, pulvinar libero. Fusce erat ex, volutpat fringilla nisl id, imperdiet tristique ligula. Ut neque sapien, feugiat non magna ut, luctus bibendum nunc. + +In hac habitasse platea dictumst. Integer vel libero eu ex efficitur interdum. Curabitur tincidunt pretium mauris, ac convallis ligula condimentum a. Cras nibh velit, varius facilisis leo non, sagittis iaculis nisi. Donec ut placerat quam, ac blandit felis. Sed ac tellus lobortis, ultrices dolor ut, luctus turpis. Vivamus vulputate ligula in nisi molestie, sit amet iaculis enim aliquet. Aenean vehicula, sem sit amet commodo imperdiet, tellus risus scelerisque libero, nec pretium nulla ante in tortor. Aliquam imperdiet urna a pharetra eleifend. Integer eget rutrum purus, quis accumsan diam. Vivamus porttitor urna nec ex cursus eleifend. Vestibulum quis sem ac justo laoreet imperdiet eu ac enim. Aliquam sed ipsum vel est venenatis vulputate eget ut sapien. Fusce id venenatis lectus. Curabitur aliquet, turpis vel semper placerat, leo arcu facilisis tellus, vel aliquet dolor dolor nec orci. Nulla condimentum velit a nisl feugiat malesuada. + +Fusce tempor nibh non augue hendrerit vehicula. Mauris eleifend metus sollicitudin quam tempus vulputate. Nam dignissim pharetra augue ac pellentesque. Phasellus ornare sit amet nisi eget laoreet. Nunc finibus molestie quam lacinia blandit. Ut gravida ligula ut tempor vestibulum. Donec vulputate turpis ligula, non dictum est congue et. Praesent mauris nibh, fringilla porttitor bibendum ut, sagittis ac nunc. Mauris eu orci sagittis, tempor mauris eget, suscipit magna. Aenean ut ex tortor. Nullam consequat, leo in interdum tincidunt, nunc felis maximus nibh, id faucibus mauris nisi id elit. Nunc auctor lorem turpis, a commodo eros luctus non. Nam ac tristique risus. Nunc pellentesque leo lectus, et mollis magna dictum sed. Suspendisse blandit quam augue, nec luctus lectus cursus a. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur iaculis pellentesque tellus in condimentum. Pellentesque volutpat convallis lorem at aliquet. Sed nec odio at ligula tempor hendrerit vitae in est. Maecenas molestie condimentum magna, aliquet bibendum sem auctor consectetur. Donec mattis, eros sed vehicula egestas, risus turpis auctor urna, vel dictum justo ex eget erat. Etiam dictum lobortis arcu, ut vestibulum ante. In a dolor sit amet risus porta hendrerit. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mauris odio, consectetur sit amet tortor sit amet, rhoncus pharetra orci. In id lectus nec tellus porttitor placerat quis vel nisi. Nam maximus neque turpis, a tincidunt metus varius vel. Aliquam congue mauris ut interdum vestibulum. Duis finibus nisl ac ultricies ullamcorper. Duis imperdiet turpis ac eros venenatis, a mollis arcu sodales. + +Nam auctor varius neque id lobortis. Pellentesque viverra malesuada viverra. Sed accumsan dui et convallis rhoncus. Aenean pellentesque pretium eros sit amet iaculis. Maecenas et massa convallis, pulvinar tortor in, sollicitudin metus. Quisque mauris nulla, eleifend eget commodo sit amet, gravida vestibulum urna. Aenean et odio eu turpis sagittis euismod. Pellentesque mollis sem turpis, a consequat erat aliquet a. + +Aenean sit amet purus elit. Donec finibus lorem quis consectetur elementum. Aliquam in cursus dolor. Fusce id volutpat quam, vitae semper enim. Duis ullamcorper leo eu ullamcorper pellentesque. Cras ipsum tellus, efficitur eget commodo ac, iaculis pharetra massa. Donec sed dolor lectus. Cras a velit quis justo placerat porta. Curabitur et nibh vel turpis porta elementum in non quam. Vivamus euismod malesuada lacus, in sollicitudin est iaculis non. Ut tincidunt nulla erat, vitae sagittis tellus vulputate eget. Vivamus nec nisl at mi bibendum hendrerit. Etiam id nunc varius, efficitur arcu non, pulvinar libero. Fusce erat ex, volutpat fringilla nisl id, imperdiet tristique ligula. Ut neque sapien, feugiat non magna ut, luctus bibendum nunc. + +In hac habitasse platea dictumst. Integer vel libero eu ex efficitur interdum. Curabitur tincidunt pretium mauris, ac convallis ligula condimentum a. Cras nibh velit, varius facilisis leo non, sagittis iaculis nisi. Donec ut placerat quam, ac blandit felis. Sed ac tellus lobortis, ultrices dolor ut, luctus turpis. Vivamus vulputate ligula in nisi molestie, sit amet iaculis enim aliquet. Aenean vehicula, sem sit amet commodo imperdiet, tellus risus scelerisque libero, nec pretium nulla ante in tortor. Aliquam imperdiet urna a pharetra eleifend. Integer eget rutrum purus, quis accumsan diam. Vivamus porttitor urna nec ex cursus eleifend. Vestibulum quis sem ac justo laoreet imperdiet eu ac enim. Aliquam sed ipsum vel est venenatis vulputate eget ut sapien. Fusce id venenatis lectus. Curabitur aliquet, turpis vel semper placerat, leo arcu facilisis tellus, vel aliquet dolor dolor nec orci. Nulla condimentum velit a nisl feugiat malesuada. + +Fusce tempor nibh non augue hendrerit vehicula. Mauris eleifend metus sollicitudin quam tempus vulputate. Nam dignissim pharetra augue ac pellentesque. Phasellus ornare sit amet nisi eget laoreet. Nunc finibus molestie quam lacinia blandit. Ut gravida ligula ut tempor vestibulum. Donec vulputate turpis ligula, non dictum est congue et. Praesent mauris nibh, fringilla porttitor bibendum ut, sagittis ac nunc. Mauris eu orci sagittis, tempor mauris eget, suscipit magna. Aenean ut ex tortor. Nullam consequat, leo in interdum tincidunt, nunc felis maximus nibh, id faucibus mauris nisi id elit. Nunc auctor lorem turpis, a commodo eros luctus non. Nam ac tristique risus. Nunc pellentesque leo lectus, et mollis magna dictum sed. Suspendisse blandit quam augue, nec luctus lectus cursus a. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur iaculis pellentesque tellus in condimentum. Pellentesque volutpat convallis lorem at aliquet. Sed nec odio at ligula tempor hendrerit vitae in est. Maecenas molestie condimentum magna, aliquet bibendum sem auctor consectetur. Donec mattis, eros sed vehicula egestas, risus turpis auctor urna, vel dictum justo ex eget erat. Etiam dictum lobortis arcu, ut vestibulum ante. In a dolor sit amet risus porta hendrerit. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mauris odio, consectetur sit amet tortor sit amet, rhoncus pharetra orci. In id lectus nec tellus porttitor placerat quis vel nisi. Nam maximus neque turpis, a tincidunt metus varius vel. Aliquam congue mauris ut interdum vestibulum. Duis finibus nisl ac ultricies ullamcorper. Duis imperdiet turpis ac eros venenatis, a mollis arcu sodales. + +Nam auctor varius neque id lobortis. Pellentesque viverra malesuada viverra. Sed accumsan dui et convallis rhoncus. Aenean pellentesque pretium eros sit amet iaculis. Maecenas et massa convallis, pulvinar tortor in, sollicitudin metus. Quisque mauris nulla, eleifend eget commodo sit amet, gravida vestibulum urna. Aenean et odio eu turpis sagittis euismod. Pellentesque mollis sem turpis, a consequat erat aliquet a. + +Aenean sit amet purus elit. Donec finibus lorem quis consectetur elementum. Aliquam in cursus dolor. Fusce id volutpat quam, vitae semper enim. Duis ullamcorper leo eu ullamcorper pellentesque. Cras ipsum tellus, efficitur eget commodo ac, iaculis pharetra massa. Donec sed dolor lectus. Cras a velit quis justo placerat porta. Curabitur et nibh vel turpis porta elementum in non quam. Vivamus euismod malesuada lacus, in sollicitudin est iaculis non. Ut tincidunt nulla erat, vitae sagittis tellus vulputate eget. Vivamus nec nisl at mi bibendum hendrerit. Etiam id nunc varius, efficitur arcu non, pulvinar libero. Fusce erat ex, volutpat fringilla nisl id, imperdiet tristique ligula. Ut neque sapien, feugiat non magna ut, luctus bibendum nunc. + +In hac habitasse platea dictumst. Integer vel libero eu ex efficitur interdum. Curabitur tincidunt pretium mauris, ac convallis ligula condimentum a. Cras nibh velit, varius facilisis leo non, sagittis iaculis nisi. Donec ut placerat quam, ac blandit felis. Sed ac tellus lobortis, ultrices dolor ut, luctus turpis. Vivamus vulputate ligula in nisi molestie, sit amet iaculis enim aliquet. Aenean vehicula, sem sit amet commodo imperdiet, tellus risus scelerisque libero, nec pretium nulla ante in tortor. Aliquam imperdiet urna a pharetra eleifend. Integer eget rutrum purus, quis accumsan diam. Vivamus porttitor urna nec ex cursus eleifend. Vestibulum quis sem ac justo laoreet imperdiet eu ac enim. Aliquam sed ipsum vel est venenatis vulputate eget ut sapien. Fusce id venenatis lectus. Curabitur aliquet, turpis vel semper placerat, leo arcu facilisis tellus, vel aliquet dolor dolor nec orci. Nulla condimentum velit a nisl feugiat malesuada. + +Fusce tempor nibh non augue hendrerit vehicula. Mauris eleifend metus sollicitudin quam tempus vulputate. Nam dignissim pharetra augue ac pellentesque. Phasellus ornare sit amet nisi eget laoreet. Nunc finibus molestie quam lacinia blandit. Ut gravida ligula ut tempor vestibulum. Donec vulputate turpis ligula, non dictum est congue et. Praesent mauris nibh, fringilla porttitor bibendum ut, sagittis ac nunc. Mauris eu orci sagittis, tempor mauris eget, suscipit magna. Aenean ut ex tortor. Nullam consequat, leo in interdum tincidunt, nunc felis maximus nibh, id faucibus mauris nisi id elit. Nunc auctor lorem turpis, a commodo eros luctus non. Nam ac tristique risus. Nunc pellentesque leo lectus, et mollis magna dictum sed. Suspendisse blandit quam augue, nec luctus lectus cursus a. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur iaculis pellentesque tellus in condimentum. Pellentesque volutpat convallis lorem at aliquet. Sed nec odio at ligula tempor hendrerit vitae in est. Maecenas molestie condimentum magna, aliquet bibendum sem auctor consectetur. Donec mattis, eros sed vehicula egestas, risus turpis auctor urna, vel dictum justo ex eget erat. Etiam dictum lobortis arcu, ut vestibulum ante. In a dolor sit amet risus porta hendrerit. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mauris odio, consectetur sit amet tortor sit amet, rhoncus pharetra orci. In id lectus nec tellus porttitor placerat quis vel nisi. Nam maximus neque turpis, a tincidunt metus varius vel. Aliquam congue mauris ut interdum vestibulum. Duis finibus nisl ac ultricies ullamcorper. Duis imperdiet turpis ac eros venenatis, a mollis arcu sodales. + +Nam auctor varius neque id lobortis. Pellentesque viverra malesuada viverra. Sed accumsan dui et convallis rhoncus. Aenean pellentesque pretium eros sit amet iaculis. Maecenas et massa convallis, pulvinar tortor in, sollicitudin metus. Quisque mauris nulla, eleifend eget commodo sit amet, gravida vestibulum urna. Aenean et odio eu turpis sagittis euismod. Pellentesque mollis sem turpis, a consequat erat aliquet a. + +Aenean sit amet purus elit. Donec finibus lorem quis consectetur elementum. Aliquam in cursus dolor. Fusce id volutpat quam, vitae semper enim. Duis ullamcorper leo eu ullamcorper pellentesque. Cras ipsum tellus, efficitur eget commodo ac, iaculis pharetra massa. Donec sed dolor lectus. Cras a velit quis justo placerat porta. Curabitur et nibh vel turpis porta elementum in non quam. Vivamus euismod malesuada lacus, in sollicitudin est iaculis non. Ut tincidunt nulla erat, vitae sagittis tellus vulputate eget. Vivamus nec nisl at mi bibendum hendrerit. Etiam id nunc varius, efficitur arcu non, pulvinar libero. Fusce erat ex, volutpat fringilla nisl id, imperdiet tristique ligula. Ut neque sapien, feugiat non magna ut, luctus bibendum nunc. + +In hac habitasse platea dictumst. Integer vel libero eu ex efficitur interdum. Curabitur tincidunt pretium mauris, ac convallis ligula condimentum a. Cras nibh velit, varius facilisis leo non, sagittis iaculis nisi. Donec ut placerat quam, ac blandit felis. Sed ac tellus lobortis, ultrices dolor ut, luctus turpis. Vivamus vulputate ligula in nisi molestie, sit amet iaculis enim aliquet. Aenean vehicula, sem sit amet commodo imperdiet, tellus risus scelerisque libero, nec pretium nulla ante in tortor. Aliquam imperdiet urna a pharetra eleifend. Integer eget rutrum purus, quis accumsan diam. Vivamus porttitor urna nec ex cursus eleifend. Vestibulum quis sem ac justo laoreet imperdiet eu ac enim. Aliquam sed ipsum vel est venenatis vulputate eget ut sapien. Fusce id venenatis lectus. Curabitur aliquet, turpis vel semper placerat, leo arcu facilisis tellus, vel aliquet dolor dolor nec orci. Nulla condimentum velit a nisl feugiat malesuada. + +Fusce tempor nibh non augue hendrerit vehicula. Mauris eleifend metus sollicitudin quam tempus vulputate. Nam dignissim pharetra augue ac pellentesque. Phasellus ornare sit amet nisi eget laoreet. Nunc finibus molestie quam lacinia blandit. Ut gravida ligula ut tempor vestibulum. Donec vulputate turpis ligula, non dictum est congue et. Praesent mauris nibh, fringilla porttitor bibendum ut, sagittis ac nunc. Mauris eu orci sagittis, tempor mauris eget, suscipit magna. Aenean ut ex tortor. Nullam consequat, leo in interdum tincidunt, nunc felis maximus nibh, id faucibus mauris nisi id elit. Nunc auctor lorem turpis, a commodo eros luctus non. Nam ac tristique risus. Nunc pellentesque leo lectus, et mollis magna dictum sed. Suspendisse blandit quam augue, nec luctus lectus cursus a. + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur iaculis pellentesque tellus in condimentum. Pellentesque volutpat convallis lorem at aliquet. Sed nec odio at ligula tempor hendrerit vitae in est. Maecenas molestie condimentum magna, aliquet bibendum sem auctor consectetur. Donec mattis, eros sed vehicula egestas, risus turpis auctor urna, vel dictum justo ex eget erat. Etiam dictum lobortis arcu, ut vestibulum ante. In a dolor sit amet risus porta hendrerit. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc mauris odio, consectetur sit amet tortor sit amet, rhoncus pharetra orci. In id lectus nec tellus porttitor placerat quis vel nisi. Nam maximus neque turpis, a tincidunt metus varius vel. Aliquam congue mauris ut interdum vestibulum. Duis finibus nisl ac ultricies ullamcorper. Duis imperdiet turpis ac eros venenatis, a mollis arcu sodales. + +Nam auctor varius neque id lobortis. Pellentesque viverra malesuada viverra. Sed accumsan dui et convallis rhoncus. Aenean pellentesque pretium eros sit amet iaculis. Maecenas et massa convallis, pulvinar tortor in, sollicitudin metus. Quisque mauris nulla, eleifend eget commodo sit amet, gravida vestibulum urna. Aenean et odio eu turpis sagittis euismod. Pellentesque mollis sem turpis, a consequat erat aliquet a. + +Aenean sit amet purus elit. Donec finibus lorem quis consectetur elementum. Aliquam in cursus dolor. Fusce id volutpat quam, vitae semper enim. Duis ullamcorper leo eu ullamcorper pellentesque. Cras ipsum tellus, efficitur eget commodo ac, iaculis pharetra massa. Donec sed dolor lectus. Cras a velit quis justo placerat porta. Curabitur et nibh vel turpis porta elementum in non quam. Vivamus euismod malesuada lacus, in sollicitudin est iaculis non. Ut tincidunt nulla erat, vitae sagittis tellus vulputate eget. Vivamus nec nisl at mi bibendum hendrerit. Etiam id nunc varius, efficitur arcu non, pulvinar libero. Fusce erat ex, volutpat fringilla nisl id, imperdiet tristique ligula. Ut neque sapien, feugiat non magna ut, luctus bibendum nunc. + +In hac habitasse platea dictumst. Integer vel libero eu ex efficitur interdum. Curabitur tincidunt pretium mauris, ac convallis ligula condimentum a. Cras nibh velit, varius facilisis leo non, sagittis iaculis nisi. Donec ut placerat quam, ac blandit felis. Sed ac tellus lobortis, ultrices dolor ut, luctus turpis. Vivamus vulputate ligula in nisi molestie, sit amet iaculis enim aliquet. Aenean vehicula, sem sit amet commodo imperdiet, tellus risus scelerisque libero, nec pretium nulla ante in tortor. Aliquam imperdiet urna a pharetra eleifend. Integer eget rutrum purus, quis accumsan diam. Vivamus porttitor urna nec ex cursus eleifend. Vestibulum quis sem ac justo laoreet imperdiet eu ac enim. Aliquam sed ipsum vel est venenatis vulputate eget ut sapien. Fusce id venenatis lectus. Curabitur aliquet, turpis vel semper placerat, leo arcu facilisis tellus, vel aliquet dolor dolor nec orci. Nulla condimentum velit a nisl feugiat malesuada. + +Fusce tempor nibh non augue hendrerit vehicula. Mauris eleifend metus sollicitudin quam tempus vulputate. Nam dignissim pharetra augue ac pellentesque. Phasellus ornare sit amet nisi eget laoreet. Nunc finibus molestie quam lacinia blandit. Ut gravida ligula ut tempor vestibulum. Donec vulputate turpis ligula, non dictum est congue et. Praesent mauris nibh, fringilla porttitor bibendum ut, sagittis ac nunc. Mauris eu orci sagittis, tempor mauris eget, suscipit magna. Aenean ut ex tortor. Nullam consequat, leo in interdum tincidunt, nunc felis maximus nibh, id faucibus mauris nisi id elit. Nunc auctor lorem turpis, a commodo eros luctus non. Nam ac tristique risus. Nunc pellentesque leo lectus, et mollis magna dictum sed. Suspendisse blandit quam augue, nec luctus lectus cursus a. diff --git a/src/test/system/fixtures/sandboxed_install/minus_one_file/sandbox.ini b/src/test/system/fixtures/sandboxed_install/minus_one_file/sandbox.ini new file mode 100644 index 0000000..a9e4774 --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/minus_one_file/sandbox.ini @@ -0,0 +1,7 @@ +[mister] +base_path = '/tmp/delme_sandbox' +base_system_path = '/tmp/delme_sandbox_system' + +[sandbox] +db_id = 'sandbox' +db_url = 'test/system/fixtures/sandboxed_install/minus_one_file/sandbox_db.json' \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/minus_one_file/sandbox_db.json b/src/test/system/fixtures/sandboxed_install/minus_one_file/sandbox_db.json new file mode 100644 index 0000000..f488f98 --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/minus_one_file/sandbox_db.json @@ -0,0 +1,18 @@ +{ + "db_files": [], + "db_id": "sandbox", + "db_url": "sandbox_db.json", + "files": { + "foo.txt": { + "delete": [], + "hash": "133af32b4894d9c5527cc5c91269ee28", + "size": 20, + "url": "https://raw.githubusercontent.com/MiSTer-devel/Downloader_MiSTer/main/src/test/system/fixtures/sandboxed_install/files/foo.txt" + } + }, + "files_count": 1, + "folders": [], + "folders_count": 0, + "latest_zip_url": "https://github.com/MiSTer-devel/Distribution_MiSTer/archive/refs/heads/main.zip", + "time": "2021/08/13 21:35:02" +} \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox.ini b/src/test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox.ini new file mode 100644 index 0000000..98ed599 --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox.ini @@ -0,0 +1,7 @@ +[mister] +base_path = '/tmp/delme_sandbox' +base_system_path = '/tmp/delme_sandbox_system' + +[sandbox] +db_id = 'sandbox' +db_url = 'test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox_db.json' \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox_db.json b/src/test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox_db.json new file mode 100644 index 0000000..a10f587 --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox_db.json @@ -0,0 +1,24 @@ +{ + "db_files": ["sandbox_db_with_extra_file.json"], + "db_id": "sandbox", + "db_url": "sandbox_db.json", + "files": { + "foo.txt": { + "delete": [], + "hash": "133af32b4894d9c5527cc5c91269ee28", + "size": 20, + "url": "https://raw.githubusercontent.com/MiSTer-devel/Downloader_MiSTer/main/src/test/system/fixtures/sandboxed_install/files/foo.txt" + }, + "bar.txt": { + "delete": [], + "hash": "942b89ab661f86228ea9ad3e980763a7", + "size": 4, + "url": "https://raw.githubusercontent.com/MiSTer-devel/Downloader_MiSTer/main/src/test/system/fixtures/sandboxed_install/files/bar.txt" + } + }, + "files_count": 1, + "folders": [], + "folders_count": 0, + "latest_zip_url": "https://github.com/MiSTer-devel/Distribution_MiSTer/archive/refs/heads/main.zip", + "time": "2021/08/13 21:35:02" +} \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox_db_with_extra_file.json b/src/test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox_db_with_extra_file.json new file mode 100644 index 0000000..6f82e5e --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox_db_with_extra_file.json @@ -0,0 +1,30 @@ +{ + "db_files": [], + "db_id": "sandbox", + "db_url": "sandbox_db.json", + "files": { + "foo.txt": { + "delete": [], + "hash": "133af32b4894d9c5527cc5c91269ee28", + "size": 20, + "url": "https://raw.githubusercontent.com/theypsilon-test/BetaDown/main/src/test/system/fixtures/sandboxed_install/files/foo.txt" + }, + "bar.txt": { + "delete": [], + "hash": "942b89ab661f86228ea9ad3e980763a7", + "size": 4, + "url": "https://raw.githubusercontent.com/theypsilon-test/BetaDown/main/src/test/system/fixtures/sandboxed_install/files/bar.txt" + }, + "baz.txt": { + "delete": [], + "hash": "ignore", + "size": 4, + "url": "https://raw.githubusercontent.com/theypsilon-test/BetaDown/main/src/test/system/fixtures/sandboxed_install/files/baz.txt" + } + }, + "files_count": 1, + "folders": [], + "folders_count": 0, + "latest_zip_url": "https://github.com/theypsilon/BetaDistrib/archive/refs/heads/main.zip", + "time": "2021/08/13 21:35:02" +} \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/sandbox.ini b/src/test/system/fixtures/sandboxed_install/sandbox.ini new file mode 100644 index 0000000..2422855 --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/sandbox.ini @@ -0,0 +1,7 @@ +[mister] +base_path = '/tmp/delme_sandbox' +base_system_path = '/tmp/delme_sandbox_system' + +[sandbox] +db_id = 'sandbox' +db_url = 'https://raw.githubusercontent.com/MiSTer-devel/Downloader_MiSTer/main/src/test/system/fixtures/sandboxed_install/sandbox_db.json' \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/sandbox_db.json b/src/test/system/fixtures/sandboxed_install/sandbox_db.json new file mode 100644 index 0000000..b20bc9f --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/sandbox_db.json @@ -0,0 +1,24 @@ +{ + "db_files": [], + "db_id": "sandbox", + "db_url": "sandbox_db.json", + "files": { + "foo.txt": { + "delete": [], + "hash": "133af32b4894d9c5527cc5c91269ee28", + "size": 20, + "url": "https://raw.githubusercontent.com/MiSTer-devel/Downloader_MiSTer/main/src/test/system/fixtures/sandboxed_install/files/foo.txt" + }, + "bar.txt": { + "delete": [], + "hash": "942b89ab661f86228ea9ad3e980763a7", + "size": 4, + "url": "https://raw.githubusercontent.com/MiSTer-devel/Downloader_MiSTer/main/src/test/system/fixtures/sandboxed_install/files/bar.txt" + } + }, + "files_count": 2, + "folders": [], + "folders_count": 0, + "latest_zip_url": "https://github.com/MiSTer-devel/Distribution_MiSTer/archive/refs/heads/main.zip", + "time": "2021/08/13 21:35:02" +} \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/updated_file/sandbox.ini b/src/test/system/fixtures/sandboxed_install/updated_file/sandbox.ini new file mode 100644 index 0000000..ab6224d --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/updated_file/sandbox.ini @@ -0,0 +1,7 @@ +[mister] +base_path = '/tmp/delme_sandbox' +base_system_path = '/tmp/delme_sandbox_system' + +[sandbox] +db_id = 'sandbox' +db_url = 'test/system/fixtures/sandboxed_install/updated_file/sandbox_db.json' \ No newline at end of file diff --git a/src/test/system/fixtures/sandboxed_install/updated_file/sandbox_db.json b/src/test/system/fixtures/sandboxed_install/updated_file/sandbox_db.json new file mode 100644 index 0000000..e4a14d5 --- /dev/null +++ b/src/test/system/fixtures/sandboxed_install/updated_file/sandbox_db.json @@ -0,0 +1,24 @@ +{ + "db_files": [], + "db_id": "sandbox", + "db_url": "sandbox_db.json", + "files": { + "foo.txt": { + "delete": [], + "hash": "133af32b4894d9c5527cc5c91269ee28", + "size": 20, + "url": "https://raw.githubusercontent.com/MiSTer-devel/Downloader_MiSTer/main/src/test/system/fixtures/sandboxed_install/files/foo.txt" + }, + "bar.txt": { + "delete": [], + "hash": "95e15b2f00a9ddec4ebbed7f33c69a52", + "size": 8, + "url": "https://raw.githubusercontent.com/MiSTer-devel/Downloader_MiSTer/main/src/test/system/fixtures/sandboxed_install/files/bar_updated.txt" + } + }, + "files_count": 2, + "folders": [], + "folders_count": 0, + "latest_zip_url": "https://github.com/MiSTer-devel/Distribution_MiSTer/archive/refs/heads/main.zip", + "time": "2021/08/13 21:35:02" +} \ No newline at end of file diff --git a/src/test/system/quick/test_sandboxed_install.py b/src/test/system/quick/test_sandboxed_install.py new file mode 100644 index 0000000..d4f9356 --- /dev/null +++ b/src/test/system/quick/test_sandboxed_install.py @@ -0,0 +1,240 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +import shutil +import os +import json +from pathlib import Path +from downloader.config import ConfigReader +from test.objects import default_env +from test.fakes import NoLogger +from downloader.file_service import hash_file, FileService +from downloader.main import main +from downloader.local_repository import LocalRepository + + +class TestSandboxedInstall(unittest.TestCase): + + sandbox_ini = "test/system/fixtures/sandboxed_install/sandbox.ini" + sandbox_db_json = 'test/system/fixtures/sandboxed_install/sandbox_db.json' + tmp_delme = '/tmp/delme_sandbox/' + + foo_file = 'test/system/fixtures/sandboxed_install/files/foo.txt' + bar_file = 'test/system/fixtures/sandboxed_install/files/bar.txt' + baz_file = 'test/system/fixtures/sandboxed_install/files/baz.txt' + + def setUp(self) -> None: + cleanup(self.sandbox_ini) + + def test_sandbox_db___installs_correctly(self): + db = load_json(self.sandbox_db_json) + self.assertExecutesCorrectly(self.sandbox_ini, { + 'local_store': local_store_files([('sandbox', db['files'])]), + 'files': hashes(self.tmp_delme, db['files']) + }) + + def test_sandbox_db___installs_correctly__twice(self): + db = load_json(self.sandbox_db_json) + self.assertExecutesCorrectly(self.sandbox_ini) + self.assertExecutesCorrectly(self.sandbox_ini, { + 'local_store': local_store_files([('sandbox', db['files'])]), + 'files': hashes(self.tmp_delme, db['files']) + }) + + def test_sandbox_db___deletes_one_installed_file_on_second_call(self): + self.assertExecutesCorrectly(self.sandbox_ini, { + 'files_count': 2 + }) + + minus_one_file_db = load_json('test/system/fixtures/sandboxed_install/minus_one_file/sandbox_db.json') + self.assertExecutesCorrectly("test/system/fixtures/sandboxed_install/minus_one_file/sandbox.ini", { + 'local_store': local_store_files([('sandbox', minus_one_file_db['files'])]), + 'files': hashes(self.tmp_delme, minus_one_file_db['files']), + 'files_count': 1 + }) + + def test_sandbox_db___updates_one_installed_file_on_second_call(self): + db = load_json(self.sandbox_db_json) + self.assertExecutesCorrectly(self.sandbox_ini, { + 'files': hashes(self.tmp_delme, db['files']), + }) + + updated_db = load_json('test/system/fixtures/sandboxed_install/updated_file/sandbox_db.json') + self.assertExecutesCorrectly("test/system/fixtures/sandboxed_install/updated_file/sandbox.ini", { + 'files': hashes(self.tmp_delme, updated_db['files']), + }) + + self.assertNotEqual(db['files']['bar.txt']['hash'], updated_db['files']['bar.txt']['hash']) + + def test_sandbox_db___updates_only_existing_files_that_changed(self): + tmp_foo_file = self.tmp_delme + '/foo.txt' + tmp_bar_file = self.tmp_delme + '/bar.txt' + + Path(tmp_foo_file).touch() + shutil.copy2(self.bar_file, self.tmp_delme) + + foo_hash_before = hash_file(tmp_foo_file) + bar_hash_before = hash_file(tmp_bar_file) + + db = load_json(self.sandbox_db_json) + self.assertExecutesCorrectly(self.sandbox_ini, { + 'local_store': local_store_files([('sandbox', db['files'])]), + 'files': hashes(self.tmp_delme, db['files']) + }) + + foo_hash_after = hash_file(tmp_foo_file) + bar_hash_after = hash_file(tmp_bar_file) + + self.assertNotEqual(foo_hash_before, foo_hash_after) + self.assertEqual(bar_hash_before, bar_hash_after) + + def test_sandbox_db___doesnt_remove_unhandled_extra_file_by_default(self): + shutil.copy2(self.foo_file, self.tmp_delme) + shutil.copy2(self.bar_file, self.tmp_delme) + shutil.copy2(self.baz_file, self.tmp_delme) + + baz_exists_before = Path(self.baz_file).is_file() + + db = load_json(self.sandbox_db_json) + files_plus_extra = db['files'].copy() + files_plus_extra['baz.txt'] = {'hash': 'c15bc5117b800187ea76873b5491e1db'} + + self.assertExecutesCorrectly(self.sandbox_ini, { + 'local_store': local_store_files([('sandbox', db['files'])]), + 'files': hashes(self.tmp_delme, files_plus_extra), + 'files_count': 3 + }) + + baz_exists_after = Path(self.baz_file).is_file() + + self.assertEqual(baz_exists_before, baz_exists_after) + + def test_sandbox_db___deletes_extra_file_from_offline_db(self): + sandbox_db_with_extra_json_file = 'test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox_db_with_extra_file.json' + + shutil.copy2(self.foo_file, self.tmp_delme) + shutil.copy2(self.bar_file, self.tmp_delme) + shutil.copy2(self.baz_file, self.tmp_delme) + shutil.copy2(sandbox_db_with_extra_json_file, self.tmp_delme) + + tmp_offline_db_file = self.tmp_delme + '/' + Path(sandbox_db_with_extra_json_file).name + tmp_baz_file = self.tmp_delme + '/' + Path(self.baz_file).name + + offline_db_exists_before = Path(tmp_offline_db_file).is_file() + baz_exists_before = Path(tmp_baz_file).is_file() + + db = load_json('test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox_db.json') + expected_local_store = local_store_files([('sandbox', db['files'])]) + expected_local_store['sandbox']['offline_databases_imported'] = ['93dd6727bc7ac342279912ad054cca64'] + + self.assertExecutesCorrectly('test/system/fixtures/sandboxed_install/offline_db_with_extra_file/sandbox.ini', { + 'local_store': expected_local_store, + 'files': hashes(self.tmp_delme, db['files']), + 'files_count': 2 + }) + + offline_db_exists_after = Path(tmp_offline_db_file).is_file() + baz_exists_after = Path(tmp_baz_file).is_file() + + self.assertNotEqual(offline_db_exists_before, offline_db_exists_after) + self.assertNotEqual(baz_exists_before, baz_exists_after) + self.assertFalse(offline_db_exists_after or baz_exists_after) + + def assertExecutesCorrectly(self, ini_path, expected=None): + self.maxDiff = None + exit_code = self.run_main(ini_path) + self.assertEqual(exit_code, 0) + + if expected is None: + return + + config = ConfigReader(NoLogger(), default_env()).read_config(ini_path) + counter = 0 + if 'local_store' in expected: + counter += 1 + actual_store = LocalRepository(config, NoLogger(), FileService(config, NoLogger())).load_store() + self.assertEqual(actual_store, expected['local_store']) + + if 'files' in expected: + counter += 1 + self.assertEqual(self.find_all(config['base_path']), expected['files']) + + if 'files_count' in expected: + counter += 1 + self.assertEqual(len(self.find_all(config['base_path'])), expected['files_count']) + + if 'system_files' in expected: + counter += 1 + self.assertEqual(self.find_all(config['base_system_path']), expected['system_files']) + + if 'system_files_count' in expected: + counter += 1 + self.assertEqual(len(self.find_all(config['base_path'])), expected['system_files_count']) + + self.assertEqual(counter, len(expected)) + + @staticmethod + def run_main(ini_path): + return main({ + 'DOWNLOADER_LAUNCHER_PATH': str(Path(ini_path).with_suffix('.sh')), + 'CURL_SSL': '', + 'UPDATE_LINUX': 'false', + 'ALLOW_REBOOT': None, + 'COMMIT': 'quick system test', + 'DEFAULT_DB_URL': '', + 'DEFAULT_DB_ID': '' + }) + + def find_all(self, directory): + return sorted(self._scan(directory), key=lambda t: t[0].lower()) + + def _scan(self, directory): + for entry in os.scandir(directory): + if entry.is_dir(follow_symlinks=False): + yield from self._scan(entry.path) + else: + yield entry.path, hash_file(entry.path) + + +def cleanup(ini_path): + config = ConfigReader(NoLogger(), default_env()).read_config(ini_path) + shutil.rmtree(config['base_path'], ignore_errors=True) + shutil.rmtree(config['base_system_path'], ignore_errors=True) + Path(config['base_path']).mkdir(parents=True, exist_ok=True) + Path(config['base_system_path']).mkdir(parents=True, exist_ok=True) + + +def local_store_files(tuples): + return { + store_id: { + 'folders': [], + 'files': files, + 'offline_databases_imported': [] + } + for store_id, files in tuples + } + + +def hashes(base_path, files): + return sorted([(base_path + f, files[f]['hash']) for f in files]) + + +def load_json(file_path): + with open(file_path, "r") as f: + return json.loads(f.read()) \ No newline at end of file diff --git a/src/test/system/slow/test_full_install.py b/src/test/system/slow/test_full_install.py new file mode 100644 index 0000000..69368a4 --- /dev/null +++ b/src/test/system/slow/test_full_install.py @@ -0,0 +1,85 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +import shutil +import os +import os.path +from pathlib import Path +from downloader.config import ConfigReader +from test.objects import default_env +from test.fakes import NoLogger +from downloader.file_service import hash_file +import subprocess + + +class TestFullInstall(unittest.TestCase): + + def test_full_install_parallel(self): + print('test_full_install_parallel') + self.assertRunOk("test/system/fixtures/full_install/parallel.ini") + + def test_full_install_serial(self): + print('test_full_install_serial') + self.assertRunOk("test/system/fixtures/full_install/serial.ini") + self.assertTrue(os.path.isfile('/tmp/delme_serial/MiSTer')) + + def test_full_install_and_rerun(self): + print('test_full_install_and_rerun A)') + self.assertRunOk("test/system/fixtures/full_install/parallel.ini") + + print('test_full_install_and_rerun B)') + self.assertRunOk("test/system/fixtures/full_install/parallel.ini") + self.assertTrue(os.path.isfile('/tmp/delme_parallel/MiSTer')) + + def test_full_install_remove_local_store_and_rerun(self): + print('test_full_install_remove_local_store_and_rerun A)') + self.assertRunOk("test/system/fixtures/full_install/parallel.ini") + + os.unlink('/tmp/delme_parallel/Scripts/.config/downloader/parallel.json.zip') + + print('test_full_install_remove_local_store_and_rerun B)') + self.assertRunOk("test/system/fixtures/full_install/parallel.ini") + self.assertTrue(os.path.isfile('/tmp/delme_parallel/MiSTer')) + + def test_full_install_remove_last_successful_run_corrupt_mister_and_rerun(self): + print('test_full_install_remove_last_successful_run_corrupt_mister_and_rerun A)') + self.assertRunOk("test/system/fixtures/full_install/parallel.ini") + + os.unlink('/tmp/delme_parallel/Scripts/.config/downloader/parallel.last_successful_run') + with open('/tmp/delme_parallel/MiSTer', 'w') as f: + f.write('corrupt') + corrupt_hash = hash_file('/tmp/delme_parallel/MiSTer') + + print('test_full_install_remove_last_successful_run_corrupt_mister_and_rerun B)') + self.assertRunOk("test/system/fixtures/full_install/parallel.ini") + correct_hash = hash_file('/tmp/delme_parallel/MiSTer') + + self.assertNotEqual(correct_hash, corrupt_hash) + + def assertRunOk(self, ini_path): + config = ConfigReader(NoLogger(), default_env()).read_config(ini_path) + shutil.rmtree(config['base_path'], ignore_errors=True) + shutil.rmtree(config['base_system_path'], ignore_errors=True) + tool = str(Path(ini_path).with_suffix('.sh')) + stem = Path(ini_path).stem + shutil.copy2('../downloader.sh', tool) + result = subprocess.run([tool], stderr=subprocess.STDOUT) + self.assertEqual(result.returncode, 0) + self.assertTrue(os.path.isfile("%s/Scripts/.config/downloader/%s.json.zip" % (config['base_system_path'], stem))) + os.unlink(tool) diff --git a/src/test/unit/__init__.py b/src/test/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/test/unit/test_config_file_path.py b/src/test/unit/test_config_file_path.py new file mode 100644 index 0000000..0251b98 --- /dev/null +++ b/src/test/unit/test_config_file_path.py @@ -0,0 +1,38 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +from downloader.config import config_file_path + + +class TestConfigFilePath(unittest.TestCase): + + def test_config_file_path___with_none___returns_downloader_ini(self): + self.assertEqual('/media/fat/downloader.ini', config_file_path(None)) + + def test_config_file_path___with_simple_relative_str___returns_str_ini(self): + self.assertEqual('./str.ini', config_file_path('str.sh')) + + def test_config_file_path___with_complex_relative_str___returns_str_ini(self): + self.assertEqual('str/complex.ini', config_file_path('str/complex.sh')) + + def test_config_file_path___with_long_custom_path___returns_downloader_ini(self): + self.assertEqual('/media/fat/custom/custom.ini', config_file_path('/media/fat/custom/custom.sh')) + + def test_config_file_path___with_long_scripts_path___returns_downloader_ini(self): + self.assertEqual('/media/fat/script.ini', config_file_path('/media/fat/Scripts/script.sh')) diff --git a/src/test/unit/test_curl_downloader.py b/src/test/unit/test_curl_downloader.py new file mode 100644 index 0000000..ac72be3 --- /dev/null +++ b/src/test/unit/test_curl_downloader.py @@ -0,0 +1,79 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +from test.fake_curl_downloader import CurlDownloader +from test.objects import file_menu_rbf, hash_menu_rbf, file_one, hash_one, file_MiSTer, hash_MiSTer, file_MiSTer_new + + +class TestCurlDownloader(unittest.TestCase): + + def setUp(self) -> None: + self.sut = CurlDownloader() + + def test_download_nothing___from_scratch_no_issues___nothing_downloaded_no_errors(self): + self.sut.download_files(False) + self.assertDownloaded([]) + + def test_download_files_one___from_scratch_no_issues___returns_correctly_downloaded_one_and_no_errors(self): + self.download_one() + self.assertDownloaded([file_one], [file_one]) + + def test_download_files_one___from_scratch_with_retry___returns_correctly_downloaded_one_and_no_errors(self): + self.sut.test_data.errors_at(file_one, 2) + self.download_one() + self.assertDownloaded([file_one], [file_one, file_one]) + + def test_download_files_one___from_scratch_could_not_download___return_errors(self): + self.sut.test_data.errors_at(file_one) + self.download_one() + self.assertDownloaded([], run=[file_one, file_one, file_one, file_one], errors=[file_one]) + + def test_download_reboot_file___from_scratch_no_issues___needs_reboot(self): + self.download_reboot() + self.assertDownloaded([file_menu_rbf], [file_menu_rbf], need_reboot=True) + + def test_download_reboot_file___update_no_issues___needs_reboot(self): + self.sut.file_service.test_data.with_file(file_menu_rbf, {'hash': 'old'}) + self.download_reboot() + self.assertDownloaded([file_menu_rbf], [file_menu_rbf], need_reboot=True) + + def test_download_reboot_file___no_changes_no_issues___no_need_to_reboot(self): + self.sut.file_service.test_data.with_file(file_menu_rbf, {'hash': hash_menu_rbf}) + self.download_reboot() + self.assertDownloaded([file_menu_rbf]) + + def test_download_mister_file___from_scratch_no_issues___stores_it_as_mister(self): + self.sut.queue_file({'url': 'https://fake.com/bar', 'hash': hash_MiSTer, 'reboot': True, 'path': 'system'}, file_MiSTer) + self.sut.download_files(False) + self.assertDownloaded([file_MiSTer], [file_MiSTer], need_reboot=True) + self.assertTrue(self.sut.file_service.is_file(file_MiSTer)) + + def assertDownloaded(self, oks, run=None, errors=None, need_reboot=False): + self.assertEqual(self.sut.correctly_downloaded_files(), oks) + self.assertEqual(self.sut.errors(), errors if errors is not None else []) + self.assertEqual(self.sut.run_files(), run if run is not None else []) + self.assertEqual(self.sut.needs_reboot(), need_reboot) + + def download_one(self): + self.sut.queue_file({'url': 'https://fake.com/bar', 'hash': hash_one}, file_one) + self.sut.download_files(False) + + def download_reboot(self): + self.sut.queue_file({'url': 'https://fake.com/bar', 'hash': hash_menu_rbf, 'reboot': True, 'path': 'system'}, file_menu_rbf) + self.sut.download_files(False) diff --git a/src/test/unit/test_linux_updater.py b/src/test/unit/test_linux_updater.py new file mode 100644 index 0000000..119dacf --- /dev/null +++ b/src/test/unit/test_linux_updater.py @@ -0,0 +1,86 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +from test.fakes import LinuxUpdater + + +class TestLinuxUpdater(unittest.TestCase): + + def test_update_linux___no_databases___no_need_to_reboot(self): + linux_updater = LinuxUpdater() + linux_updater.update_linux() + self.assertFalse(linux_updater.needs_reboot()) + self.assertEqual(linux_updater.file_service.read_file_contents('/MiSTer.version'), "unknown") + + def test_update_linux___no_linux_databases___no_need_to_reboot(self): + linux_updater = LinuxUpdater() + linux_updater.add_db({'db_id': 'first'}) + linux_updater.add_db({'db_id': 'second'}) + linux_updater.update_linux() + self.assertFalse(linux_updater.needs_reboot()) + self.assertEqual(linux_updater.file_service.read_file_contents('/MiSTer.version'), "unknown") + + def test_update_linux___db_with_new_linux___has_new_version_and_needs_reboot(self): + linux_updater = LinuxUpdater() + linux_updater.add_db({'db_id': 'new', 'linux': linux_description()}) + linux_updater.update_linux() + self.assertTrue(linux_updater.needs_reboot()) + self.assertEqual(linux_updater.file_service.read_file_contents('/MiSTer.version'), "210711") + + def test_update_linux___db_with_old_linux___has_old_version_and_no_need_to_reboot(self): + linux_updater = LinuxUpdater() + linux_updater.file_service.test_data.with_file('/MiSTer.version', {'content': "210711"}) + linux_updater.add_db({'db_id': 'new', 'linux': linux_description()}) + linux_updater.update_linux() + self.assertFalse(linux_updater.needs_reboot()) + self.assertEqual(linux_updater.file_service.read_file_contents('/MiSTer.version'), "210711") + + def test_update_linux___dbs_with_different_new_linux___updates_first_linux_and_needs_reboot(self): + linux_updater = LinuxUpdater() + linux_updater.add_db({'db_id': 'new_2', 'linux': linux_description_with_version("222222")}) + linux_updater.add_db({'db_id': 'new_1', 'linux': linux_description_with_version("111111")}) + linux_updater.add_db({'db_id': 'new_3', 'linux': linux_description_with_version("333333")}) + linux_updater.update_linux() + self.assertTrue(linux_updater.needs_reboot()) + self.assertEqual(linux_updater.file_service.read_file_contents('/MiSTer.version'), "222222") + + def test_update_linux___new_linux_but_failed_download___no_need_to_reboot(self): + linux_updater = LinuxUpdater() + linux_updater.downloader.test_data.errors_at('linux.7z') + linux_updater.add_db({'db_id': 'new', 'linux': linux_description()}) + linux_updater.update_linux() + self.assertFalse(linux_updater.needs_reboot()) + self.assertEqual(linux_updater.file_service.read_file_contents('/MiSTer.version'), "unknown") + + +def linux_description(): + return linux_description_with_version("210711") + + +def linux_description_with_version(version): + return { + "delete": [], + "hash": "d3b619c54c4727ab618bf108013f79d9", + "size": 83873790, + "url": linux_url, + "version": version + } + + +linux_url = "https://raw.githubusercontent.com/MiSTer-devel/SD-Installer-Win64_MiSTer/136d7d8ea24b1de2424574b2d31f527d6b3e3d39/release_20210711.rar" diff --git a/src/test/unit/test_offline_importer.py b/src/test/unit/test_offline_importer.py new file mode 100644 index 0000000..a55640f --- /dev/null +++ b/src/test/unit/test_offline_importer.py @@ -0,0 +1,104 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +from downloader.other import empty_store +from test.objects import db_test_with_file_a_descr, file_test_json_zip_descr, file_test_json_zip, file_a +from test.fakes import OfflineImporter + + +class TestOfflineImporter(unittest.TestCase): + + def setUp(self) -> None: + self.sut = OfflineImporter() + + def test_apply_offline_databases___for_test_db_when_a_file_is_present_with_correct_hash___adds_existing_a_file_to_the_store(self): + self.sut.file_service.test_data\ + .with_test_json_zip()\ + .with_file_a() + + store = self.apply_db_test_with_file_a() + self.assertTrue(file_a in store['files']) + self.assertFalse(self.sut.file_service.is_file(file_test_json_zip)) + + def test_apply_offline_databases___for_test_db_when_a_file_is_present_with_incorrect_hash___adds_nothing_to_the_store(self): + self.sut.file_service.test_data\ + .with_test_json_zip()\ + .with_file_a({'hash': 'incorrect'}) + + store = self.apply_db_test_with_file_a() + self.assertFalse(file_a in store['files']) + self.assertFalse(self.sut.file_service.is_file(file_test_json_zip)) + + def test_apply_offline_databases___for_test_db_when_a_file_is_present_with_ignore_hash___adds_existing_a_file_to_the_store(self): + json_zip_db = file_test_json_zip_descr() + json_zip_db['unzipped_json']['files'][file_a]['hash'] = 'ignore' + self.sut.file_service.test_data\ + .with_test_json_zip(json_zip_db)\ + .with_file_a({'hash': 'incorrect'}) + + store = self.apply_db_test_with_file_a() + self.assertTrue(file_a in store['files']) + self.assertFalse(self.sut.file_service.is_file(file_test_json_zip)) + + def test_apply_offline_databases___when_empty___adds_nothing_to_the_store(self): + self.sut.file_service.test_data.with_test_json_zip() + + store = self.apply_db_test_with_file_a() + self.assertFalse(file_a in store['files']) + self.assertFalse(self.sut.file_service.is_file(file_test_json_zip)) + + def test_apply_offline_databases___always___adds_db_hash_to_offline_databases_imported(self): + self.sut.file_service.test_data.with_test_json_zip() + + store = self.apply_db_test_with_file_a() + self.assertTrue(file_test_json_zip in store['offline_databases_imported']) + self.assertFalse(self.sut.file_service.is_file(file_test_json_zip)) + + def test_apply_offline_databases___when_db_has_file_a_but_is_already_at_offline_databases_imported___just_deletes_db_file(self): + self.sut.file_service.test_data\ + .with_test_json_zip()\ + .with_file_a() + + store = empty_store() + store['offline_databases_imported'].append(file_test_json_zip) + self.sut.add_db(db_test_with_file_a_descr(), store) + self.sut.apply_offline_databases() + self.assertFalse(file_a in store['files']) + self.assertFalse(self.sut.file_service.is_file(file_test_json_zip)) + + def test_apply_offline_databases___when_file_does_not_exist___does_nothing(self): + store = self.apply_db_test_with_file_a() + self.assertTrue(file_test_json_zip not in store['offline_databases_imported']) + self.assertEqual(store, empty_store()) + self.assertFalse(self.sut.file_service.is_file(file_test_json_zip)) + + def test_apply_offline_databases___when_db_id_does_not_match___does_nothing(self): + self.sut.file_service.test_data\ + .with_test_json_zip({'hash': file_test_json_zip, 'unzipped_json': {'db_id': 'does_not_match'}}) + + store = self.apply_db_test_with_file_a() + self.assertTrue(file_test_json_zip not in store['offline_databases_imported']) + self.assertEqual(store, empty_store()) + self.assertTrue(self.sut.file_service.is_file(file_test_json_zip)) + + def apply_db_test_with_file_a(self): + store = empty_store() + self.sut.add_db(db_test_with_file_a_descr(), store) + self.sut.apply_offline_databases() + return store diff --git a/src/test/unit/test_online_importer.py b/src/test/unit/test_online_importer.py new file mode 100644 index 0000000..1a21063 --- /dev/null +++ b/src/test/unit/test_online_importer.py @@ -0,0 +1,248 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +from downloader.config import default_config +from downloader.other import empty_store +from test.objects import db_empty_descr, file_boot_rom, boot_rom_descr, overwrite_file, file_mister_descr, file_a_descr, file_a_updated_descr, db_test_with_file, db_with_file, db_test_with_file_a_descr, file_a, file_MiSTer +from test.fakes import OnlineImporter + + +class TestOnlineImporter(unittest.TestCase): + + def setUp(self) -> None: + self.sut = OnlineImporter() + + def test_download_dbs_contents___with_trivial_db___does_nothing(self): + sut = OnlineImporter() + store = empty_store() + + sut.add_db(db_empty_descr(), store) + sut.download_dbs_contents(False) + + self.assertReportsNothing(sut) + self.assertEqual(store, empty_store()) + + def test_download_dbs_contents___being_empty___does_nothing(self): + sut = OnlineImporter() + sut.download_dbs_contents(False) + self.assertReportsNothing(sut) + + def test_download_dbs_contents___with_one_file___fills_store_with_that_file(self): + sut = OnlineImporter() + store = empty_store() + + sut.add_db(db_test_with_file_a_descr(), store) + sut.download_dbs_contents(False) + + self.assertEqual(store['files'][file_a], file_a_descr()) + self.assertReports(sut, [file_a]) + self.assertTrue(sut.file_service.is_file(file_a)) + + def test_download_dbs_contents___with_existing_incorrect_file_but_correct_already_on_store___changes_nothing(self): + sut = OnlineImporter() + sut.file_service.test_data.with_file_a({'hash': 'does_not_match'}) + store = db_test_with_file_a_descr() + + sut.add_db(db_test_with_file_a_descr(), store) + sut.download_dbs_contents(False) + + self.assertEqual(store['files'][file_a], file_a_descr()) + self.assertReportsNothing(sut) + self.assertEqual(sut.file_service.hash(file_a), 'does_not_match') + + def test_download_dbs_contents___with_existing_incorrect_file_also_on_store___downloads_the_correct_one(self): + sut = OnlineImporter() + sut.file_service.test_data.with_file_a({'hash': 'does_not_match'}) + store = db_test_with_file(file_a, {'hash': 'does_not_match'}) + + sut.add_db(db_test_with_file_a_descr(), store) + sut.download_dbs_contents(False) + + self.assertEqual(store['files'][file_a], file_a_descr()) + self.assertReports(sut, [file_a]) + self.assertEqual(sut.file_service.hash(file_a), file_a) + + def test_download_dbs_contents___with_non_existing_one_file_already_on_store___installs_file_regardless(self): + sut = OnlineImporter() + store = db_test_with_file_a_descr() + + sut.add_db(db_test_with_file_a_descr(), store) + sut.download_dbs_contents(False) + + self.assertEqual(store['files'][file_a], file_a_descr()) + self.assertReports(sut, [file_a]) + self.assertTrue(sut.file_service.is_file(file_a)) + + def test_download_dbs_contents___with_one_failed_file___just_reports_error(self): + sut = OnlineImporter() + sut.downloader_test_data.errors_at(file_a) + store = empty_store() + + sut.add_db(db_test_with_file_a_descr(), store) + sut.download_dbs_contents(False) + + self.assertEqual(store['files'], {}) + self.assertReports(sut, [], errors=[file_a]) + self.assertFalse(sut.file_service.is_file(file_a)) + + def test_download_dbs_contents___with_mister___needs_reboot(self): + sut = OnlineImporter() + store = empty_store() + + sut.add_db(db_test_with_file(file_MiSTer, file_mister_descr()), store) + sut.download_dbs_contents(False) + + self.assertEqual(store['files'][file_MiSTer], file_mister_descr()) + self.assertReports(sut, [file_MiSTer], needs_reboot=True) + self.assertTrue(sut.file_service.is_file(file_MiSTer)) + + def test_download_dbs_contents___with_file_on_stored_erroring___store_deletes_file(self): + sut = OnlineImporter() + sut.downloader_test_data.errors_at(file_a) + store = db_test_with_file_a_descr() + + sut.add_db(db_test_with_file(file_a, file_a_updated_descr()), store) + sut.download_dbs_contents(False) + + self.assertEqual(store['files'], {}) + self.assertReports(sut, [], errors=[file_a]) + self.assertFalse(sut.file_service.is_file(file_a)) + + def test_download_dbs_contents___with_duplicated_file___just_accounts_for_the_first_added(self): + sut = OnlineImporter() + store = empty_store() + + sut.add_db(db_with_file('test', file_a, file_a_descr()), store) + sut.add_db(db_with_file('bar', file_a, file_a_updated_descr()), store) + sut.download_dbs_contents(False) + + self.assertEqual(store['files'][file_a], file_a_descr()) + self.assertReports(sut, [file_a]) + self.assertEqual(sut.file_service.hash(file_a), file_a_descr()['hash']) + + def test_download_dbs_contents___when_file_a_gets_removed___store_becomes_empty(self): + sut = OnlineImporter() + sut.file_service.test_data.with_file_a() + store = db_test_with_file_a_descr() + + sut.add_db(db_empty_descr(), store) + sut.download_dbs_contents(False) + + self.assertEqual(store['files'], {}) + self.assertReportsNothing(sut) + self.assertFalse(sut.file_service.is_file(file_a)) + + def test_download_dbs_contents___when_file_is_already_there___does_nothing(self): + sut = OnlineImporter() + sut.file_service.test_data.with_file_a() + store = db_test_with_file_a_descr() + + sut.add_db(db_test_with_file_a_descr(), store) + sut.download_dbs_contents(False) + + self.assertReportsNothing(sut) + self.assertTrue(sut.file_service.is_file(file_a)) + + def test_download_dbs_contents___when_downloaded_file_is_missing___downloads_it_again(self): + sut = OnlineImporter() + store = db_test_with_file_a_descr() + + sut.add_db(db_test_with_file_a_descr(), store) + sut.download_dbs_contents(False) + + self.assertReports(sut, [file_a]) + self.assertTrue(sut.file_service.is_file(file_a)) + + def test_download_dbs_contents___when_no_check_downloaded_files_and_downloaded_file_is_missing___does_nothing(self): + config = default_config() + config['check_manually_deleted_files'] = False + sut = OnlineImporter(config=config) + store = db_test_with_file_a_descr() + + sut.add_db(db_test_with_file_a_descr(), store) + sut.download_dbs_contents(False) + + self.assertReportsNothing(sut) + self.assertFalse(sut.file_service.is_file(file_a)) + + def test_overwrite___when_boot_rom_present___should_not_overwrite_it(self): + sut = OnlineImporter() + sut.file_service.test_data.with_file(file_boot_rom, {"hash": "something_else"}) + + sut.add_db(db_test_with_file(file_boot_rom, boot_rom_descr()), empty_store()) + sut.download_dbs_contents(False) + + self.assertReportsNothing(sut) + self.assertEqual(sut.file_service.hash(file_boot_rom), "something_else") + + def test_overwrite___when_boot_rom_present_but_with_different_case___should_not_overwrite_it(self): + sut = OnlineImporter() + sut.file_service.test_data.with_file(file_boot_rom.upper(), {"hash": "something_else"}) + + sut.add_db(db_test_with_file(file_boot_rom.lower(), boot_rom_descr()), empty_store()) + sut.download_dbs_contents(False) + + self.assertReportsNothing(sut) + self.assertEqual(sut.file_service.hash(file_boot_rom.lower()), "something_else") + self.assertNotEqual(sut.file_service.hash(file_boot_rom.lower()), boot_rom_descr()['hash']) + + def test_overwrite___when_overwrite_yes_file_a_is_present___should_overwrite_it(self): + sut = OnlineImporter() + sut.file_service.test_data.with_file_a() + + sut.add_db(db_test_with_file(file_a, overwrite_file(file_a_updated_descr(), True)), empty_store()) + sut.download_dbs_contents(False) + + self.assertReports(sut, [file_a]) + self.assertEqual(sut.file_service.hash(file_a), file_a_updated_descr()['hash']) + self.assertNotEqual(sut.file_service.hash(file_a), file_a_descr()['hash']) + + def test_overwrite___when_overwrite_no_file_a_is_present___should_overwrite_it(self): + sut = OnlineImporter() + sut.file_service.test_data.with_file_a() + + sut.add_db(db_test_with_file(file_a, overwrite_file(file_a_updated_descr(), False)), empty_store()) + sut.download_dbs_contents(False) + + self.assertReportsNothing(sut) + self.assertEqual(sut.file_service.hash(file_a), file_a_descr()['hash']) + self.assertNotEqual(sut.file_service.hash(file_a), file_a_updated_descr()['hash']) + + def test_overwrite___when_file_a_without_overwrite_is_present___should_overwrite_it(self): + sut = OnlineImporter() + sut.file_service.test_data.with_file_a() + + sut.add_db(db_test_with_file(file_a, file_a_updated_descr()), empty_store()) + sut.download_dbs_contents(False) + + self.assertReports(sut, [file_a]) + self.assertEqual(sut.file_service.hash(file_a), file_a_updated_descr()['hash']) + self.assertNotEqual(sut.file_service.hash(file_a), file_a_descr()['hash']) + + def assertReportsNothing(self, sut): + self.assertReports(sut, []) + + def assertReports(self, sut, installed, errors=None, needs_reboot=False): + if errors is None: + errors = [] + if installed is None: + installed = [] + self.assertEqual(sut.correctly_installed_files(), installed) + self.assertEqual(sut.files_that_failed(), errors) + self.assertEqual(sut.needs_reboot(), needs_reboot) diff --git a/src/test/unit/test_reboot_calculator.py b/src/test/unit/test_reboot_calculator.py new file mode 100644 index 0000000..3f91d70 --- /dev/null +++ b/src/test/unit/test_reboot_calculator.py @@ -0,0 +1,60 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +from test.fakes import RebootCalculator +from test.fake_file_service import FileService +from downloader.config import AllowReboot +from downloader.reboot_calculator import mister_downloader_needs_reboot_file + + +class TestRebootCalculator(unittest.TestCase): + + def test_calc_needs_reboot___when_nothing_needs_reboot___returns_false_and_doesnt_create_reboot_file(self): + fs = FileService() + actual = RebootCalculator(file_service=fs).calc_needs_reboot(False, False) + self.assertFalse(actual) + self.assertFalse(fs.is_file(mister_downloader_needs_reboot_file)) + + def test_calc_needs_reboot___when_linux_needs_reboot___returns_true(self): + actual = RebootCalculator().calc_needs_reboot(True, False) + self.assertTrue(actual) + + def test_calc_needs_reboot___when_importer_needs_reboot___returns_true(self): + actual = RebootCalculator().calc_needs_reboot(False, True) + self.assertTrue(actual) + + def test_calc_needs_reboot___when_everything_needs_reboot___returns_true(self): + actual = RebootCalculator().calc_needs_reboot(True, True) + self.assertTrue(actual) + + def test_calc_needs_reboot___when_no_reboot_config_but_everything_needs_reboot___returns_false_and_creates_reboot_file(self): + fs = FileService() + actual = RebootCalculator({'allow_reboot': AllowReboot.NEVER}, fs).calc_needs_reboot(True, True) + self.assertFalse(actual) + self.assertTrue(fs.is_file(mister_downloader_needs_reboot_file)) + + def test_calc_needs_reboot___when_only_linux_reboots_and_importer_needs_reboot___returns_false_and_creates_reboot_file(self): + fs = FileService() + actual = RebootCalculator({'allow_reboot': AllowReboot.ONLY_AFTER_LINUX_UPDATE}, fs).calc_needs_reboot(False, True) + self.assertFalse(actual) + self.assertTrue(fs.is_file(mister_downloader_needs_reboot_file)) + + def test_calc_needs_reboot___when_only_linux_reboots_and_linux_needs_reboot___returns_true(self): + actual = RebootCalculator({'allow_reboot': AllowReboot.ONLY_AFTER_LINUX_UPDATE}).calc_needs_reboot(True, False) + self.assertTrue(actual) diff --git a/src/test/unit/test_run_downloader.py b/src/test/unit/test_run_downloader.py new file mode 100644 index 0000000..34d37ff --- /dev/null +++ b/src/test/unit/test_run_downloader.py @@ -0,0 +1,128 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +from downloader.main import run_downloader +from test.fakes import LocalRepository, NoLogger, OnlineImporter, OfflineImporter, LinuxUpdater +from test.fake_db_gateway import DbGateway + + +class TestRunDownloader(unittest.TestCase): + def test_run_downloader___no_databases___returns_0(self): + exit_code = run_downloader( + {'COMMIT': 'test', 'UPDATE_LINUX': 'false'}, + {'databases': []}, + NoLogger(), + LocalRepository(), + DbGateway(), + OfflineImporter(), + OnlineImporter(), + LinuxUpdater() + ) + self.assertEqual(exit_code, 0) + + def test_run_downloader___empty_databases___returns_0(self): + exit_code = run_downloader( + {'COMMIT': 'test', 'UPDATE_LINUX': 'false'}, + {'databases': [{ + 'db_url': 'empty', + 'section': 'empty' + }]}, + NoLogger(), + LocalRepository(), + DbGateway({'empty': { + 'db_id': 'empty', + 'db_files': [], + 'files': [], + 'folders': [] + }}), + OfflineImporter(), + OnlineImporter(), + LinuxUpdater() + ) + self.assertEqual(exit_code, 0) + + def test_run_downloader___database_with_new_linux___returns_0(self): + linux_updater = LinuxUpdater() + exit_code = run_downloader( + {'COMMIT': 'test', 'UPDATE_LINUX': 'false'}, + {'databases': [{ + 'db_url': 'empty', + 'section': 'empty' + }]}, + NoLogger(), + LocalRepository(), + DbGateway({'empty': empty_db_with_linux()}), + OfflineImporter(), + OnlineImporter(), + linux_updater + ) + self.assertEqual(exit_code, 0) + + def test_run_downloader___database_with_wrong_id___returns_1(self): + exit_code = run_downloader( + {'COMMIT': 'test', 'UPDATE_LINUX': 'false'}, + {'databases': [{ + 'db_url': 'empty', + 'section': 'empty' + }]}, + NoLogger(), + LocalRepository(), + DbGateway({'empty': { + 'db_id': 'wrong', + 'db_files': [], + 'files': [], + 'folders': [] + }}), + OfflineImporter(), + OnlineImporter(), + LinuxUpdater() + ) + self.assertEqual(exit_code, 1) + + def test_run_downloader___database_not_fetched___returns_1(self): + exit_code = run_downloader( + {'COMMIT': 'test', 'UPDATE_LINUX': 'false'}, + {'databases': [{ + 'db_url': 'empty', + 'section': 'empty' + }]}, + NoLogger(), + LocalRepository(), + DbGateway(), + OfflineImporter(), + OnlineImporter(), + LinuxUpdater() + ) + self.assertEqual(exit_code, 1) + + +def empty_db_with_linux(): + return { + 'db_id': 'empty', + 'db_files': [], + 'files': [], + 'folders': [], + 'linux': { + "delete": [], + "hash": "d3b619c54c4727ab618bf108013f79d9", + "size": 83873790, + "url": "https://raw.githubusercontent.com/MiSTer-devel/SD-Installer-Win64_MiSTer/136d7d8ea24b1de2424574b2d31f527d6b3e3d39/release_20210711.rar", + "version": "210711" + } + } diff --git a/src/test/unit/test_smoke.py b/src/test/unit/test_smoke.py new file mode 100644 index 0000000..e90e586 --- /dev/null +++ b/src/test/unit/test_smoke.py @@ -0,0 +1,26 @@ +# Copyright (c) 2021 José Manuel Barroso Galindo + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# You can download the latest version of this tool from: +# https://github.com/MiSTer-devel/Downloader_MiSTer + +import unittest +import __main__ + + +class TestSmoke(unittest.TestCase): + + def test_true(self): + self.assertTrue(True)