fixes apt-update issue 100 where Suites changed, kills update sound t… (#422)

* fixes apt-update issue 100 where Suites changed, kills update sound thread, sets updating to false if synchronization fails

* cleans up code

* appmgr changes

* ui changes

* updates required appmgr in agent

* actually upgrade current version in appmgr

.gitignore

@@ -1,3 +1,4 @@
+.DS_Store
 /*.img
 /buster.zip
 /product_key

BuildGuide.md

@@ -0,0 +1,179 @@
+##### Initial Notes & Recommendations
+* Due to issues to cross-compile the image from a desktop, this guide will take you step-by-step through the process of compiling EmbassyOS directly on a Raspberry Pi 4 (4GB or 8GB)
+* This process will go faster if you have an SSD/NVMe USB drive available.
+* This build guide does **not** require a large microSD card, especially if your final build wil be used on an SSD/NVMe USB drive.
+* Basic know-how of linux commands and terminal use is recommended.
+* Follow the guide carefully and do not skip any steps.
+
+# :hammer_and_wrench: Build Guide
+1. Flash [Raspberry Pi OS Lite](https://www.raspberrypi.org/software/operating-systems/) to a microSD and configure your raspi to boot from SSD/NVMe USB drive
+   1. After flashing, create an empty text file called `ssh` in the `boot` partition of the microSD, then proceed with booting the raspi with the flashed microSD (check your router for the IP assigned to your raspi)
+   1. Do the usual initial update/config
+      ```
+      sudo apt update
+      sudo raspi-config
+      ```
+   1. Change `Advanced Options->Boot Order`
+   1. Select `USB Boot` *(it will try to boot from microSD first if it's available)*
+   1. Select `Finish`, then `Yes` to reboot
+   1. After reboot, `sudo shutdown now` to power off the raspi and remove the microSD
+  
+2. Flash the *Raspi OS Lite* (from step 1) to your SSD/NVMe drive
+   > :information_source: Don't worry about rootfs partition size (raspi will increase it for you on initial boot)
+   
+   > :information_source: Every time you re-flash your SSD/NVMe you need to first boot with a microSD and set *Boot Order* again
+
+   1. Don't forget to create the empty `ssh` file
+   1. Connect the drive (remember to remove the microSD) to the raspi and start it up
+   1. Use `sudo raspi-config` to change the default password
+   1. Optional: `sudo apt upgrade -y`
+   1. Optional: `sudo nano /etc/apt/sources.list.d/vscode.list` comment the last line which contains `packages.microsoft.com`
+
+3. Install GHC
+   ```
+   sudo apt update
+   sudo apt install -y ghc
+   
+   #test:
+   ghc --version
+   
+   #example of output:
+   The Glorious Glasgow Haskell Compilation System, version 8.4.4
+   ```
+
+4. Compile Stack:
+   1. Install Stack v2.1.3
+      ```
+      cd ~/
+      wget -qO- https://raw.githubusercontent.com/commercialhaskell/stack/v2.1.3/etc/scripts/get-stack.sh | sh
+      
+      #test with
+      stack --version
+      
+      #example output:
+      Version 2.1.3, Git revision 636e3a759d51127df2b62f90772def126cdf6d1f (7735 commits) arm hpack-0.31.2
+      ```
+    
+   1. Use current Stack to compile Stack v2.5.1:
+      ```
+      git clone --depth 1 --branch v2.5.1 https://github.com/commercialhaskell/stack.git
+      cd stack
+      sudo apt install -y screen
+      screen
+      ```
+      > :information_source: Build (>=3.5h total... We are using `screen` in case of session timeout issues)
+     
+      > :memo:  If you get disconected you can reattach last sesion again by executing `screen -r`
+      ```
+      stack build --stack-yaml=stack-ghc-84.yaml --system-ghc
+      
+      #Install
+      stack install --stack-yaml=stack-ghc-84.yaml --system-ghc
+      export PATH=~/.local/bin:$PATH
+      ```
+
+5. Clone EmbassyOS & try to *make* the `agent`:
+   1. First attempt
+      > :information_source: The first time you run **make** you'll get an error
+      
+      ```
+      sudo apt install -y llvm-9 libgmp-dev
+      export PATH=/usr/lib/llvm-9/bin:$PATH
+      cd ~/
+      git clone https://github.com/Start9Labs/embassy-os.git
+      cd embassy-os/
+      make agent
+      ```
+      > :memo: This will install ghc-8.10.2, then attempt to build but will give errors (in next steps we deal with errors)
+   1. Confirm your cpu info
+      ```
+      cat /proc/cpuinfo | grep Hardware
+      ```
+   1. If your "Hardware" is [BCM2711](https://www.raspberrypi.org/documentation/hardware/raspberrypi/bcm2711/README.md) then:
+      1. Change `C compiler flags` to `-marm -fno-stack-protector -mcpu=cortex-a7` in the GHC settings:
+         ```
+         nano ~/.stack/programs/arm-linux/ghc-8.10.4/lib/ghc-8.10.4/settings
+         ```
+   1. To prevent gcc errors we delete the `setup-exe-src` folder
+      ```
+      rm -rf ~/.stack/setup-exe-src/
+      ```
+   1. Re-make the agent
+      ```
+      make agent
+      ```
+
+6. Install requirements for step 7
+   1. Install NVM
+      ```
+      cd ~/ && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash
+      export NVM_DIR="$HOME/.nvm"
+      [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"  # This loads nvm
+      [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion"  # This loads nvm bash_completion
+      nvm --version
+      ```
+   1. Install Node.js & NPM
+      ```
+      nvm install node
+      ```
+   1. Install Ionic CLI
+      ```
+      npm install -g @ionic/cli
+      ```
+   1. Install Dependencies
+      ```
+      sudo apt-get install -y build-essential openssl libssl-dev libc6-dev clang libclang-dev libavahi-client-dev upx ca-certificates
+      ```
+   1. Install Rust
+      ```
+      cd ~/ && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs -o- | bash
+      
+      #Choose option 1
+      source $HOME/.cargo/env
+
+      #Check rust & cargo versions
+      rustc --version
+      cargo --version
+      ```
+
+7. Finally, getting to build the **.img**
+   1. At this stage you hava a working development environment to build your **embassy.img**.
+      Before you do that you can choose to enable SSH login for user `pi` in case something will go wrong or just skip to the next step.
+      ```
+      cd ~/embassy-os
+      sed -e '/passwd -l pi/ s/^#*/#/' -i setup.sh
+      ```
+      > :warning: Default password for user `pi` is `raspberry`, change it the next you login.
+   1. Build the `embassy.img`
+      ```
+      cd ~/embassy-os
+      make
+      
+      #Depending on your hardware this can take 1-2h+
+      #Wait for the "DONE!" message and take note of your product_key
+      exit
+      ```
+8. Flash the `embassy.img` to a microSD
+   1. Copy `embassy.img` from the raspi to your PC with scp
+      ```
+      scp pi@raspi_IP:~/embassy-os/embassy.img .
+      ```
+   1. Connect to raspi again to do `sudo shutdown now`, after a complete shutdown disconnect SSD/NVMe drive
+   1. Flash `embassy.img` to a microSD (do this before flashing to the SSD/NVMe, to be sure it works)
+
+9. Prepare for initial setup
+   1. Boot raspi using flashed microSD
+   1. After a few minutes, the raspi should reboot itself and make it's first [sounds](#embassy-sounds-explained).
+      > :information_source: If needed, you can check the `agent` log with: `journalctl -u agent -ef`
+   1. Proceed with the [initial setup process of EmbassyOS](https://docs.start9labs.com/user-manual/initial-setup.html)
+   1. If all went well you can safely flash `embassy.img` to an SSD/NVMe and repeat step 9
+
+### Embassy sounds explained
+Sound :notes: | Indicating 
+------- | --------
+Bep | Device is powering on
+Chime | Device is ready for setup
+Mario "Coin" | EmbassyOS has started
+Mario "Death" | Device is about to Shutdown/Reboot
+Mario "Power Up" | EmbassyOS update sequence
+Beethoven | Update failed :(

CONTRIBUTING.md

@@ -0,0 +1,237 @@
+<!-- omit in toc -->
+# Contributing to Embassy OS
+
+First off, thanks for taking the time to contribute! ❤️
+
+All types of contributions are encouraged and valued. See the [Table of Contents](#table-of-contents) for different ways to help and details about how this project handles them. Please make sure to read the relevant section before making your contribution. It will make it a lot easier for us maintainers and smooth out the experience for all involved. The community looks forward to your contributions. 🎉
+
+> And if you like the project, but just don't have time to contribute, that's fine. There are other easy ways to support the project and show your appreciation, which we would also be very happy about:
+> - Star the project
+> - Tweet about it
+> - Refer this project in your project's readme
+> - Mention the project at local meetups and tell your friends/colleagues
+> - Buy an [Embassy](https://start9labs.com)
+
+<!-- omit in toc -->
+## Table of Contents
+
+- [I Have a Question](#i-have-a-question)
+- [I Want To Contribute](#i-want-to-contribute)
+  - [Reporting Bugs](#reporting-bugs)
+  - [Suggesting Enhancements](#suggesting-enhancements)
+  - [Your First Code Contribution](#your-first-code-contribution)
+    - [Setting Up Your Development Environment](#setting-up-your-development-environment)
+    - [Building The Image](#building-the-image)
+  - [Improving The Documentation](#improving-the-documentation)
+- [Styleguides](#styleguides)
+  - [Formatting](#formatting)
+  - [Atomic Commits](#atomic-commits)
+  - [Commit Messages](#commit-messages)
+  - [Pull Requests](#pull-requests)
+  - [Rebasing Changes](#rebasing-changes)
+- [Join The Discussion](#join-the-discussion)
+- [Join The Project Team](#join-the-project-team)
+
+
+
+## I Have a Question
+
+> If you want to ask a question, we assume that you have read the available [Documentation](https://docs.start9labs.com).
+
+Before you ask a question, it is best to search for existing [Issues](https://github.com/Start9Labs/embassy-os/issues) that might help you. In case you have found a suitable issue and still need clarification, you can write your question in this issue. It is also advisable to search the internet for answers first.
+
+If you then still feel the need to ask a question and need clarification, we recommend the following:
+
+- Open an [Issue](https://github.com/Start9Labs/embassy-os/issues/new).
+- Provide as much context as you can about what you're running into.
+- Provide project and platform versions, depending on what seems relevant.
+
+We will then take care of the issue as soon as possible.
+
+<!--
+You might want to create a separate issue tag for questions and include it in this description. People should then tag their issues accordingly.
+
+Depending on how large the project is, you may want to outsource the questioning, e.g. to Stack Overflow or Gitter. You may add additional contact and information possibilities:
+- IRC
+- Slack
+- Gitter
+- Stack Overflow tag
+- Blog
+- FAQ
+- Roadmap
+- E-Mail List
+- Forum
+-->
+
+## I Want To Contribute
+
+> ### Legal Notice <!-- omit in toc -->
+> When contributing to this project, you must agree that you have authored 100% of the content, that you have the necessary rights to the content and that the content you contribute may be provided under the project license.
+
+### Reporting Bugs
+
+<!-- omit in toc -->
+#### Before Submitting a Bug Report
+
+A good bug report shouldn't leave others needing to chase you up for more information. Therefore, we ask you to investigate carefully, collect information and describe the issue in detail in your report. Please complete the following steps in advance to help us fix any potential bug as fast as possible.
+
+- Make sure that you are using the latest version.
+- Determine if your bug is really a bug and not an error on your side e.g. using incompatible environment components/versions (Make sure that you have read the [documentation](https://docs.start9labs.com). If you are looking for support, you might want to check [this section](#i-have-a-question)).
+- To see if other users have experienced (and potentially already solved) the same issue you are having, check if there is not already a bug report existing for your bug or error in the [bug tracker](https://github.com/Start9Labs/embassy-os/issues?q=label%3Abug).
+- Also make sure to search the internet (including Stack Overflow) to see if users outside of the GitHub community have discussed the issue.
+- Collect information about the bug:
+  - Stack trace (Traceback)
+  - Client OS, Platform and Version (Windows/Linux/macOS/iOS/Android, Firefox/Tor Browser/Consulate)
+  - Version of the interpreter, compiler, SDK, runtime environment, package manager, depending on what seems relevant.
+  - Possibly your input and the output
+  - Can you reliably reproduce the issue? And can you also reproduce it with older versions?
+
+<!-- omit in toc -->
+#### How Do I Submit a Good Bug Report?
+
+> You must never report security related issues, vulnerabilities or bugs to the issue tracker, or elsewhere in public. Instead sensitive bugs must be sent by email to <security@start9labs.com>.
+<!-- You may add a PGP key to allow the messages to be sent encrypted as well. -->
+
+We use GitHub issues to track bugs and errors. If you run into an issue with the project:
+
+- Open an [Issue](https://github.com/Start9Labs/embassy-os/issues/new). (Since we can't be sure at this point whether it is a bug or not, we ask you not to talk about a bug yet and not to label the issue.)
+- Explain the behavior you would expect and the actual behavior.
+- Please provide as much context as possible and describe the *reproduction steps* that someone else can follow to recreate the issue on their own. This usually includes your code. For good bug reports you should isolate the problem and create a reduced test case.
+- Provide the information you collected in the previous section.
+
+Once it's filed:
+
+- The project team will label the issue accordingly.
+- A team member will try to reproduce the issue with your provided steps. If there are no reproduction steps or no obvious way to reproduce the issue, the team will ask you for those steps and mark the issue as `needs-repro`. Bugs with the `needs-repro` tag will not be addressed until they are reproduced.
+- If the team is able to reproduce the issue, it will be marked `needs-fix`, as well as possibly other tags (such as `critical`), and the issue will be left to be [implemented by someone](#your-first-code-contribution).
+
+<!-- You might want to create an issue template for bugs and errors that can be used as a guide and that defines the structure of the information to be included. If you do so, reference it here in the description. -->
+
+
+### Suggesting Enhancements
+
+This section guides you through submitting an enhancement suggestion for Embassy OS, **including completely new features and minor improvements to existing functionality**. Following these guidelines will help maintainers and the community to understand your suggestion and find related suggestions.
+
+<!-- omit in toc -->
+#### Before Submitting an Enhancement
+
+- Make sure that you are using the latest version.
+- Read the [documentation](https://docs.start9labs.com) carefully and find out if the functionality is already covered, maybe by an individual configuration.
+- Perform a [search](https://github.com/Start9Labs/embassy-os/issues) to see if the enhancement has already been suggested. If it has, add a comment to the existing issue instead of opening a new one.
+- Find out whether your idea fits with the scope and aims of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Keep in mind that we want features that will be useful to the majority of our users and not just a small subset. If you're just targeting a minority of users, consider writing an add-on/plugin library.
+
+<!-- omit in toc -->
+#### How Do I Submit a Good Enhancement Suggestion?
+
+Enhancement suggestions are tracked as [GitHub issues](https://github.com/Start9Labs/embassy-os/issues).
+
+- Use a **clear and descriptive title** for the issue to identify the suggestion.
+- Provide a **step-by-step description of the suggested enhancement** in as many details as possible.
+- **Describe the current behavior** and **explain which behavior you expected to see instead** and why. At this point you can also tell which alternatives do not work for you.
+- You may want to **include screenshots and animated GIFs** which help you demonstrate the steps or point out the part which the suggestion is related to. You can use [this tool](https://www.cockos.com/licecap/) to record GIFs on macOS and Windows, and [this tool](https://github.com/colinkeenan/silentcast) or [this tool](https://github.com/GNOME/byzanz) on Linux. <!-- this should only be included if the project has a GUI -->
+- **Explain why this enhancement would be useful** to most Embassy OS users. You may also want to point out the other projects that solved it better and which could serve as inspiration.
+
+<!-- You might want to create an issue template for enhancement suggestions that can be used as a guide and that defines the structure of the information to be included. If you do so, reference it here in the description. -->
+
+### Project Structure
+Embassy OS has 3 main components: `agent`, `appmgr`, and `ui`.
+- The `ui` (Typescript Ionic Angular) is the code that is deployed to the browser to provide the user interface for Embassy OS
+- The `agent` (Haskell) is a daemon that provides the interface for the `ui` to interact with the Embassy, as well as manage system state.
+- `appmgr` (Rust) is a command line utility and (soon to be) daemon that sets up and manages services and their environments.
+
+### Your First Code Contribution
+
+#### Setting up your development environment
+##### agent
+There are two main workflows to consider when developing on the agent. During the development process you will spend
+most of your time developing in an environment where you cannot actually run the agent. This is because we make heavy
+platform specific assumptions (by nature of the project) around what folders get used and what package management tools
+are used for the underlying system. If you are running this on a platform besides Linux you won't even be able to run
+the agent effectively on your dev machine. Even if you are on Linux you may not want to turn administrative control over
+to the software you are currently developing. So how do you know that anything you are doing is right? We make extensive
+use of Haskell's type system and surrounding tooling. For this you will want to make sure you are using the [haskell-language-server](https://github.com/haskell/haskell-language-server)
+and [stack](https://github.com/commercialhaskell/stack)
+
+At some point though you will want to build the agent for the target platform (Raspberry Pi 4). This is the second build
+flow that you will need to consider.
+
+At Start9 we build the agent in two different ways. The primary way we have done it is on the Raspberry Pi itself. To do
+this you will need stack built for the Raspberry Pi. Unfortunately, however, FPComplete no longer
+distributes ARMv7 binaries for stack. Though hopefully soon we will be able to submit the binaries we've built for this
+project back to them and have them hosted more visibly. The way we bootstrap through this problem is by downloading version
+[2.1.3](https://github.com/commercialhaskell/stack/releases/download/v2.1.3/stack-2.1.3-linux-arm.tar.gz) and using that
+to compile v2.5.1. Before you can successfully compile anything with GHC on the Raspberry Pi. You will need to tweak the
+relevant GHC config. You will need to edit the file at `~/.stack/programs/arm-linux/ghc-8.10.2/lib/ghc-8.10.2/settings`
+and change the line `("C compiler flags", " -marm -fno-stack-protector -mcpu=cortex-a7")` to include `-mcpu=cortex-a7`.
+You will also need to make sure you've downloaded and installed LLVM 9.
+
+Once you have done these things, you simply need to `cd` into the embassy-os project and then run `make agent`.
+
+##### ui
+- Requirements
+  - [Install nodejs](https://nodejs.org/en/)
+  - [Install npm](https://www.npmjs.com/get-npm)
+  - [Install ionic cli](https://ionicframework.com/docs/intro/cli)
+- Scripts (run within ./ui directory)
+  - `npm i` installs ui node package dependencies
+  - `npm run build` compiles project, depositing build artifacts into ./ui/www
+  - `npm run build-prod` as above but customized for deployment to an Embassy
+  - `ionic serve` serves the ui on localhost:8100 for local development. Edit ./ui/use-mocks.json to 'true' to use mocks during local development
+  - `./build-send.sh <embassy .local address suffix>` builds the project and deploys it to the referenced Embassy
+    - Find your Embassy on the LAN using the Start9 Setup App or network tools. It's address will be of the form `start9-<suffix>.local`.
+    - For example `./build-send.sh abcdefgh` will deploy the ui to the Embassy with LAN address `start9-abcdefgh.local`.
+    - SSH keys must be installed on the Embassy prior to running this script.
+
+##### appmgr
+  - [Install Rust](https://rustup.rs)
+  - Recommended: [rust-analyzer](https://rust-analyzer.github.io/)
+
+#### Building The Image
+- Requirements
+  - `ext4fs` (available if running on the Linux kernel)
+  - [Docker](https://docs.docker.com/get-docker/)
+  - GNU Make
+- Building
+  - build the [agent](#agent)
+    - make sure resulting artifact is agent/dist/agent
+  - run `make`
+
+### Improving The Documentation
+You can find the repository for Start9's documentation [here](https://github.com/Start9Labs/documentation). If there is something you would like to see added, let us know, or create an issue yourself. Welcome are contributions for lacking or incorrect information, broken links, requested additions, or general style improvements. 
+
+Contributions in the form of setup guides for integrations with external applications are highly encouraged. If you struggled through a process and would like to share your steps with others, check out the docs for each [service](https://github.com/Start9Labs/documentation/blob/master/source/user-manuals/available-services/index.rst) we support. The wrapper repos contain sections for adding integration guides, such as this [one](https://github.com/Start9Labs/bitcoind-wrapper/tree/master/docs). These not only help out others in the community, but inform how we can create a more seamless and intuitive experience.
+
+## Styleguides
+### Formatting
+Code must be formatted with the formatter designated for each component:
+- `ui`: [tslint](https://palantir.github.io/tslint/)
+- `agent`: [brittany](https://github.com/lspitzner/brittany)
+- `appmgr`: [rustfmt](https://github.com/rust-lang/rustfmt)
+
+### Atomic Commits
+Commits [should be atomic](https://en.wikipedia.org/wiki/Atomic_commit#Atomic_commit_convention) and diffs should be easy to read.
+Do not mix any formatting fixes or code moves with actual code changes.
+
+### Commit Messages
+If a commit touches only 1 component, prefix the message with the affected component. i.e. `appmgr: update to tokio v0.3`.
+
+### Pull Requests
+The body of a pull request should contain sufficient description of what the changes do, as well as a justification.
+You should include references to any relevant [issues](https://github.com/Start9Labs/embassy-os/issues).
+
+### Rebasing Changes
+When a pull request conflicts with the target branch, you may be asked to rebase it on top of the current target branch. The git rebase command will take care of rebuilding your commits on top of the new base.
+
+This project aims to have a clean git history, where code changes are only made in non-merge commits. This simplifies auditability because merge commits can be assumed to not contain arbitrary code changes.
+
+## Join The Discussion
+Current or aspiring contributors? Join our community developer [Matrix channel](https://matrix.to/#/#community-dev:matrix.start9labs.com).
+
+Just interested in or using the project? Join our community [Telegram](https://t.me/start9_labs) or [Matrix](https://matrix.to/#/#community:matrix.start9labs.com).
+
+## Join The Project Team
+Interested in becoming a part of the Start9 Labs team? Send an email to <jobs@start9labs.com>
+
+<!-- omit in toc -->
+## Attribution
+This guide is based on the **contributing-gen**. [Make your own](https://github.com/bttger/contributing-gen)!

LICENSE.md

@@ -2,17 +2,24 @@
 
 This license governs the use of the accompanying Software. If you use the Software, you accept this license. If you do not accept the license, do not use the Software.
 
-1. **Definitions**
+1. **Definitions.**
-    1. "Licensor" means the copyright owner, Start9 Labs, Inc, or its successor(s) in interest, or a future assignee of the copyright.
+    1. “Licensor” means the copyright owner, Start9 Labs, Inc, or its successor(s) in interest, or a future assignee of the copyright.
-    2. "Source Code" means the code made available to you by the Licensor pursuant to the terms of this license and any derivative works based thereon.
+    2. “Source Code” means the preferred form of the Software for making modifications to it.
-    3. "Object Code" means any non-source form of the Source Code, including the machine-language output by a compiler or assembler.
+    3. “Object Code” means any non-source form of the Software, including the machine-language output by a compiler or assembler.
-    4. "Distribute" means to convey or to publish and generally has the same meaning here as under U.S. Copyright law.
+    4. “Distribute” means to convey or to publish and generally has the same meaning here as under U.S. Copyright law.
-    5. "Personal Use" means accessing, copying, reviewing, auditing, running, testing, or modifying the Source Code.
+    5. “Sell” means practicing any or all of the rights granted to you under the License to provide to third parties, for a fee or other consideration (including without limitation fees for hosting or consulting/support services related to the Software), a product or service whose value derives, entirely or substantially, from the functionality of the Software.
-2. **Grant of Rights**
+
-    1. Subject to the terms of this license, the Licensor grants you, the licensee, a non-exclusive, worldwide, royalty-free copyright license to the Source Code for Personal Use only.
+2. **Grant of Rights.** Subject to the terms of this license, the Licensor grants you, the licensee, a non-exclusive, worldwide, royalty-free copyright license to:
-    2. Subject to the terms of this license, the Licensor grants you, the licensee, the right to Distribute the Source Code or modifications to the Source Code.
+    1. Access, audit, copy, modify, compile, or distribute the Source Code or modifications to the Source Code.
-    3. Distributing the Object Code, or any Object Code created based on your modifications to the Source Code, is not permitted under the terms of this license without express written consent of the Licensor.
+    2. Run, test, or otherwise use the Object Code.
-    4. If you Distribute the Source Code, or if permission is granted to Distribute the Object Code, you expressly undertake not to remove, or modify, in any manner, the copyright notices attached to the Source Code, and displayed in any output of the Object Code when run, and to reproduce these notices, in an identical manner, in any distributed copies of the Source Code or Object Code together with a copy of this license. If you Distribute a modified copy of the Software or a derivative work based thereon, the work must carry prominent notices stating that you modified it, and giving a relevant date.
+
-    5. The terms of this license will apply to anyone who comes into possession of a copy of the Source Code or Object Code, and any modifications or derivative works based thereon, made by anyone.
+3. **Limitations.**
-3. **Disclaimer.** THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. Licensor has no obligation to support recipients of the Source Code or Object Code.
+    1. The grant of rights under the License will NOT include, and the License does NOT grant you the right to:
+        1. Sell the Software or any derivative works based thereon.
+        2. Distribute the Object Code.
+    2. If you Distribute the Source Code, or if permission is separately granted to Distribute the Object Code, you expressly undertake not to remove, or modify, in any manner, the copyright notices attached to the Source Code, and displayed in any output of the Object Code when run, and to reproduce these notices, in an identical manner, in any distributed copies of the Software together with a copy of this license. If you Distribute a modified copy of the Software, or a derivative work based thereon, the work must carry prominent notices stating that you modified it, and giving a relevant date.
+    3. The terms of this license will apply to anyone who comes into possession of a copy of the Software, and any modifications or derivative works based thereon, made by anyone.
+
 4. **Contributions.** You hereby grant to Licensor a perpetual, irrevocable, worldwide, non-exclusive, royalty-free license to use and exploit any modifications or derivative works based on the Source Code of which you are the author.
+
+5. **Disclaimer.** THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. LICENSOR HAS NO OBLIGATION TO SUPPORT RECIPIENTS OF THE SOFTWARE.

Makefile

@@ -1,12 +1,33 @@
-APPMGR_SRC := $(shell find appmgr/src) appmgr/Cargo.toml appmgr/Cargo.lock
+UNAME := $(shell uname -m)
-AGENT_SRC := $(shell find agent/src) $(shell find agent/config) agent/stack.yaml agent/package.yaml agent/build.sh
 
-.DELETE_ON_ERROR:
+EMBASSY_SRC := buster.img product_key appmgr/target/armv7-unknown-linux-gnueabihf/release/appmgr ui/www agent/dist/agent agent/config/agent.service lifeline/target/armv7-unknown-linux-gnueabihf/release/lifeline lifeline/lifeline.service setup.sh setup.service docker-daemon.json
+APPMGR_RELEASE_SRC := appmgr/target/armv7-unknown-linux-gnueabihf/release/appmgr
+LIFELINE_RELEASE_SRC := lifeline/target/armv7-unknown-linux-gnueabihf/release/lifeline
+
+ifeq ($(UNAME), armv7l)
+	EMBASSY_SRC := buster.img product_key appmgr/target/release/appmgr ui/www agent/dist/agent agent/config/agent.service lifeline/target/release/lifeline lifeline/lifeline.service setup.sh setup.service docker-daemon.json
+	APPMGR_RELEASE_SRC := appmgr/target/release/appmgr
+	LIFELINE_RELEASE_SRC := lifeline/target/release/lifeline
+endif
+
+APPMGR_SRC := $(shell find appmgr/src) appmgr/Cargo.toml appmgr/Cargo.lock
+LIFELINE_SRC := $(shell find lifeline/src) lifeline/Cargo.toml lifeline/Cargo.lock
+AGENT_SRC := $(shell find agent/src) $(shell find agent/config) agent/stack.yaml agent/package.yaml agent/build.sh
+UI_SRC := $(shell find ui/src) \
+			ui/angular.json \
+			ui/browserslist \
+			ui/client-manifest.yaml \
+			ui/ionic.config.json \
+			ui/postprocess.ts \
+			ui/tsconfig.json \
+			ui/tslint.json \
+			ui/use-mocks.json
 
 all: embassy.img
 
-embassy.img: buster.img product_key appmgr/target/armv7-unknown-linux-musleabihf/release/appmgr ui/www agent/dist/agent agent/config/agent.service
+embassy.img: $(EMBASSY_SRC)
-	./make_image.sh
+	chmod +x make_image.sh
+	sudo ./make_image.sh
 
 buster.img:
 	wget -O buster.zip https://downloads.raspberrypi.org/raspios_lite_armhf/images/raspios_lite_armhf-2020-08-24/2020-08-20-raspios-buster-armhf-lite.zip
@@ -18,9 +39,37 @@ product_key:
 	echo "X\c" > product_key
 	cat /dev/random | base32 | head -c11 | tr '[:upper:]' '[:lower:]' >> product_key
 
-appmgr/target/armv7-unknown-linux-musleabihf/release/appmgr: $(APPMGR_SRC)
+$(APPMGR_RELEASE_SRC): $(APPMGR_SRC)
-	docker run --rm -it -v ~/.cargo/registry:/root/.cargo/registry -v "$(shell pwd)"/appmgr:/home/rust/src start9/rust-arm-cross:latest cargo build --release --features=production
+ifeq ($(UNAME), armv7l)
-	docker run --rm -it -v ~/.cargo/registry:/root/.cargo/registry -v "$(shell pwd)"/appmgr:/home/rust/src start9/rust-arm-cross:latest arm-linux-gnueabi-strip target/armv7-unknown-linux-gnueabihf/release/appmgr
+	cd appmgr && cargo update && cargo build --release --features=production
+	arm-linux-gnueabihf-strip appmgr/target/release/appmgr
+else
+	docker run --rm -it -v ~/.cargo/registry:/root/.cargo/registry -v "$(shell pwd)":/home/rust/src start9/rust-arm-cross:latest sh -c "(cd appmgr && cargo build --release --features=production)"
+	docker run --rm -it -v ~/.cargo/registry:/root/.cargo/registry -v "$(shell pwd)":/home/rust/src start9/rust-arm-cross:latest arm-linux-gnueabi-strip appmgr/target/armv7-unknown-linux-gnueabihf/release/appmgr
+endif
 
-agent: $(AGENT_SRC)
+appmgr: $(APPMGR_RELEASE_SRC)
-	(cd agent; ./build.sh)
+
+agent/dist/agent: $(AGENT_SRC)
+	(cd agent && ./build.sh)
+
+agent: agent/dist/agent
+
+ui/node_modules: ui/package.json
+	npm --prefix ui install
+
+ui/www: $(UI_SRC) ui/node_modules
+	npm --prefix ui run build-prod
+
+ui: ui/www
+
+$(LIFELINE_RELEASE_SRC): $(LIFELINE_SRC)
+ifeq ($(UNAME), armv7l)
+	cd lifeline && cargo build --release
+	arm-linux-gnueabihf-strip lifeline/target/release/lifeline
+else
+	docker run --rm -it -v ~/.cargo/registry:/root/.cargo/registry -v "$(shell pwd)":/home/rust/src start9/rust-arm-cross:latest sh -c "(cd lifeline && cargo build --release)"
+	docker run --rm -it -v ~/.cargo/registry:/root/.cargo/registry -v "$(shell pwd)":/home/rust/src start9/rust-arm-cross:latest arm-linux-gnueabi-strip lifeline/target/armv7-unknown-linux-gnueabihf/release/lifeline
+endif
+
+lifeline: $(LIFELINE_RELEASE_SRC)

README.md

@@ -1 +1,47 @@
-# Embassy OS
+# EmbassyOS
+[![Version](https://img.shields.io/github/v/tag/Start9Labs/embassy-os?color=success)](https://github.com/Start9Labs/embassy-os/releases)
+[![community](https://img.shields.io/badge/community-matrix-yellow)](https://matrix.to/#/#community:matrix.start9labs.com)
+[![community](https://img.shields.io/badge/community-telegram-informational)](https://t.me/start9_labs)
+[![support](https://img.shields.io/badge/support-docs-important)](https://docs.start9labs.com)
+[![developer](https://img.shields.io/badge/developer-matrix-blueviolet)](https://matrix.to/#/#community-dev:matrix.start9labs.com)
+[![website](https://img.shields.io/website?down_color=lightgrey&down_message=offline&up_color=green&up_message=online&url=https%3A%2F%2Fstart9labs.com)](https://start9labs.com)
+
+[![mastodon](https://img.shields.io/mastodon/follow/000000001?domain=https%3A%2F%2Fmastodon.start9labs.com&label=Follow&style=social)](http://mastodon.start9labs.com)
+[![twitter](https://img.shields.io/twitter/follow/start9labs?label=Follow)](https://twitter.com/start9labs)
+
+### _Anyone can do it. No one can stop it._ ###
+
+EmbassyOS is a mass-market, graphical operating system designed to facilitate the discovery, installation, configuration, private self-hosting, and reliable operation of open-source software services and applications. It aims to eliminate trust and custodianship from personal computing.
+
+<img src="assets/eos.png" width="100%">
+
+## :warning: Caution
+Some technologies supported by this software, such as [Lightning](https://lightning.network/), are considered in active development and might experience issues. Do not commit any funds you are not willing to lose. Be #reckless at your own risk. 
+
+## Running EmbassyOS
+There are multiple ways to obtain and begin using EmbassyOS.
+
+### :moneybag: Buy an Embassy
+This is the most convenient option. Simply [buy an Embassy](https://start9labs.com) from Start9 Labs and plug it in. Depending on where you live, shipping costs and import duties may vary.
+
+### :construction_worker: Build your own Embassy
+While not as convenient as buying an Embassy, this option is easier than you might imagine, and there are 4 reasons why you might prefer it:
+1. You already have a Raspberry Pi and would like to re-purpose it.
+1. You want to save on shipping costs.
+1. You prefer not to divulge your physical shipping address.
+1. You just like building things.
+
+To pursue this option, follow this [guide](https://docs.start9labs.com/getting-started/diy.html).
+
+### :hammer_and_wrench: Build EmbassyOS from Source
+
+EmbassyOS can be built from source, for personal use, for free.
+A detailed guide for doing so can be found [here](https://github.com/Start9Labs/embassy-os/blob/master/BuildGuide.md).
+
+## :heart: Contributing
+To contribute to the development of EmbassyOS, see [here](https://github.com/Start9Labs/embassy-os/blob/master/CONTRIBUTING.md).
+
+## UI Screenshots
+<img src="assets/ServicesRunning.png" alt="Embassy Services" width="100%"> | <img src="assets/ServiceDetails.png" alt="Service Details" width="100%">
+--- | ---
+<img src="assets/Embassy.png" alt="EmbassyOS" width="100%"> | <img src="assets/Marketplace.png" alt="Marketplace" width="100%">

agent/.gitignore

@@ -19,9 +19,7 @@ cabal.sandbox.config
 *.keter
 *~
 .vscode
-*.cabal
 \#*
-start9-companion-server.cabal
 stack.yaml.lock
 *.env
 agent_*

agent/ambassador-agent.cabal

@@ -0,0 +1,521 @@
+cabal-version: 1.12
+
+-- This file has been generated from package.yaml by hpack version 0.34.4.
+--
+-- see: https://github.com/sol/hpack
+
+name:           ambassador-agent
+version:        0.2.16
+build-type:     Simple
+extra-source-files:
+    ./migrations/0.1.0::0.1.0
+    ./migrations/0.1.0::0.1.1
+    ./migrations/0.1.1::0.1.2
+    ./migrations/0.1.2::0.1.3
+    ./migrations/0.1.3::0.1.4
+    ./migrations/0.1.4::0.1.5
+    ./migrations/0.1.5::0.2.0
+    ./migrations/0.2.0::0.2.1
+    ./migrations/0.2.10::0.2.11
+    ./migrations/0.2.11::0.2.12
+    ./migrations/0.2.12::0.2.13
+    ./migrations/0.2.13::0.2.14
+    ./migrations/0.2.14::0.2.15
+    ./migrations/0.2.15::0.2.16
+    ./migrations/0.2.1::0.2.2
+    ./migrations/0.2.2::0.2.3
+    ./migrations/0.2.3::0.2.4
+    ./migrations/0.2.4::0.2.5
+    ./migrations/0.2.5::0.2.6
+    ./migrations/0.2.6::0.2.7
+    ./migrations/0.2.7::0.2.8
+    ./migrations/0.2.8::0.2.9
+    ./migrations/0.2.9::0.2.10
+
+flag dev
+  description: Turn on development settings, like auto-reload templates.
+  manual: False
+  default: False
+
+flag disable-auth
+  description: disable authorization checks
+  manual: False
+  default: False
+
+flag library-only
+  description: Build for use with "yesod devel"
+  manual: False
+  default: False
+
+library
+  exposed-modules:
+      Application
+      Auth
+      Constants
+      Daemon.AppNotifications
+      Daemon.RefreshProcDev
+      Daemon.SslRenew
+      Daemon.TorHealth
+      Daemon.ZeroConf
+      Foundation
+      Handler.Apps
+      Handler.Authenticate
+      Handler.Backups
+      Handler.Hosts
+      Handler.Icons
+      Handler.Login
+      Handler.Network
+      Handler.Notifications
+      Handler.PasswordUpdate
+      Handler.PowerOff
+      Handler.Register
+      Handler.Register.Nginx
+      Handler.Register.Tor
+      Handler.SelfUpdate
+      Handler.SshKeys
+      Handler.Status
+      Handler.Tor
+      Handler.Types.Apps
+      Handler.Types.HmacSig
+      Handler.Types.Hosts
+      Handler.Types.Metrics
+      Handler.Types.Parse
+      Handler.Types.Register
+      Handler.Types.V0.Base
+      Handler.Types.V0.Specs
+      Handler.Types.V0.Ssh
+      Handler.Types.V0.Wifi
+      Handler.Util
+      Handler.V0
+      Handler.Wifi
+      Lib.Algebra.Domain.AppMgr
+      Lib.Algebra.Domain.AppMgr.TH
+      Lib.Algebra.Domain.AppMgr.Types
+      Lib.Algebra.State.RegistryUrl
+      Lib.Avahi
+      Lib.Background
+      Lib.ClientManifest
+      Lib.Crypto
+      Lib.Database
+      Lib.Error
+      Lib.External.AppManifest
+      Lib.External.AppMgr
+      Lib.External.Metrics.Df
+      Lib.External.Metrics.Iotop
+      Lib.External.Metrics.ProcDev
+      Lib.External.Metrics.Temperature
+      Lib.External.Metrics.Top
+      Lib.External.Metrics.Types
+      Lib.External.Registry
+      Lib.External.Specs.Common
+      Lib.External.Specs.CPU
+      Lib.External.Specs.Memory
+      Lib.External.Util
+      Lib.External.WpaSupplicant
+      Lib.IconCache
+      Lib.Metrics
+      Lib.Migration
+      Lib.Notifications
+      Lib.Password
+      Lib.ProductKey
+      Lib.SelfUpdate
+      Lib.Sound
+      Lib.Ssh
+      Lib.Ssl
+      Lib.Synchronizers
+      Lib.SystemCtl
+      Lib.SystemPaths
+      Lib.Tor
+      Lib.TyFam.ConditionalData
+      Lib.Types.Core
+      Lib.Types.Emver
+      Lib.Types.Emver.Orphans
+      Lib.Types.NetAddress
+      Lib.Types.ServerApp
+      Lib.Types.Url
+      Lib.WebServer
+      Model
+      Orphans.Digest
+      Orphans.UUID
+      Settings
+      Startlude
+      Startlude.ByteStream
+      Startlude.ByteStream.Char8
+      Util.Conduit
+      Util.File
+      Util.Function
+      Util.Text
+  other-modules:
+      Paths_ambassador_agent
+  hs-source-dirs:
+      src
+  default-extensions:
+      NoImplicitPrelude
+      BlockArguments
+      ConstraintKinds
+      DataKinds
+      DeriveAnyClass
+      DeriveFunctor
+      DeriveGeneric
+      DerivingStrategies
+      EmptyCase
+      FlexibleContexts
+      FlexibleInstances
+      GADTs
+      GeneralizedNewtypeDeriving
+      InstanceSigs
+      KindSignatures
+      LambdaCase
+      MultiParamTypeClasses
+      MultiWayIf
+      NamedFieldPuns
+      NumericUnderscores
+      OverloadedStrings
+      PolyKinds
+      RankNTypes
+      StandaloneDeriving
+      StandaloneKindSignatures
+      TupleSections
+      TypeApplications
+      TypeFamilies
+      TypeOperators
+  build-depends:
+      aeson
+    , aeson-flatten
+    , attoparsec
+    , base >=4.9.1.0 && <5
+    , bytestring
+    , casing
+    , comonad
+    , conduit
+    , conduit-extra
+    , connection
+    , containers
+    , cryptonite
+    , cryptonite-conduit
+    , data-default
+    , directory
+    , errors
+    , exceptions
+    , exinst
+    , fast-logger
+    , file-embed
+    , filelock
+    , filepath
+    , fused-effects
+    , fused-effects-th
+    , git-embed
+    , http-api-data
+    , http-client
+    , http-client-tls
+    , http-conduit
+    , http-types
+    , interpolate
+    , iso8601-time
+    , json-rpc
+    , lens
+    , lens-aeson
+    , lifted-async
+    , lifted-base
+    , memory
+    , mime-types
+    , monad-control
+    , monad-logger
+    , network
+    , persistent
+    , persistent-sqlite
+    , persistent-template
+    , process
+    , process-extras
+    , protolude
+    , regex-compat
+    , resourcet
+    , shell-conduit
+    , singletons
+    , stm
+    , streaming
+    , streaming-bytestring
+    , streaming-conduit
+    , streaming-utils
+    , tar-conduit
+    , template-haskell
+    , text >=0.11 && <2.0
+    , time
+    , transformers
+    , transformers-base
+    , typed-process
+    , unix
+    , unliftio
+    , unliftio-core
+    , unordered-containers
+    , uuid
+    , wai
+    , wai-cors
+    , wai-extra
+    , warp
+    , yaml
+    , yesod
+    , yesod-auth
+    , yesod-core
+    , yesod-form
+    , yesod-persistent
+  if (flag(dev)) || (flag(library-only))
+    ghc-options: -Wall -Wunused-packages -fwarn-tabs -O0 -fdefer-typed-holes
+    cpp-options: -DDEVELOPMENT
+  else
+    ghc-options: -Wall -Wunused-packages -fwarn-tabs -O2 -fdefer-typed-holes
+  if (flag(disable-auth))
+    cpp-options: -DDISABLE_AUTH
+  default-language: Haskell2010
+
+executable agent
+  main-is: main.hs
+  hs-source-dirs:
+      app
+  default-extensions:
+      NoImplicitPrelude
+      BlockArguments
+      ConstraintKinds
+      DataKinds
+      DeriveAnyClass
+      DeriveFunctor
+      DeriveGeneric
+      DerivingStrategies
+      EmptyCase
+      FlexibleContexts
+      FlexibleInstances
+      GADTs
+      GeneralizedNewtypeDeriving
+      InstanceSigs
+      KindSignatures
+      LambdaCase
+      MultiParamTypeClasses
+      MultiWayIf
+      NamedFieldPuns
+      NumericUnderscores
+      OverloadedStrings
+      PolyKinds
+      RankNTypes
+      StandaloneDeriving
+      StandaloneKindSignatures
+      TupleSections
+      TypeApplications
+      TypeFamilies
+      TypeOperators
+  ghc-options: -Wall -threaded -rtsopts -with-rtsopts=-N -fdefer-typed-holes
+  build-depends:
+      aeson
+    , aeson-flatten
+    , ambassador-agent
+    , attoparsec
+    , base >=4.9.1.0 && <5
+    , bytestring
+    , casing
+    , comonad
+    , conduit
+    , conduit-extra
+    , connection
+    , containers
+    , cryptonite
+    , cryptonite-conduit
+    , data-default
+    , directory
+    , errors
+    , exceptions
+    , exinst
+    , fast-logger
+    , file-embed
+    , filelock
+    , filepath
+    , fused-effects
+    , fused-effects-th
+    , git-embed
+    , http-api-data
+    , http-client
+    , http-client-tls
+    , http-conduit
+    , http-types
+    , interpolate
+    , iso8601-time
+    , json-rpc
+    , lens
+    , lens-aeson
+    , lifted-async
+    , lifted-base
+    , memory
+    , mime-types
+    , monad-control
+    , monad-logger
+    , network
+    , persistent
+    , persistent-sqlite
+    , persistent-template
+    , process
+    , process-extras
+    , protolude
+    , regex-compat
+    , resourcet
+    , shell-conduit
+    , singletons
+    , stm
+    , streaming
+    , streaming-bytestring
+    , streaming-conduit
+    , streaming-utils
+    , tar-conduit
+    , template-haskell
+    , text >=0.11 && <2.0
+    , time
+    , transformers
+    , transformers-base
+    , typed-process
+    , unix
+    , unliftio
+    , unliftio-core
+    , unordered-containers
+    , uuid
+    , wai
+    , wai-cors
+    , wai-extra
+    , warp
+    , yaml
+    , yesod
+    , yesod-auth
+    , yesod-core
+    , yesod-form
+    , yesod-persistent
+  if flag(library-only)
+    buildable: False
+  default-language: Haskell2010
+
+test-suite agent-test
+  type: exitcode-stdio-1.0
+  main-is: Main.hs
+  other-modules:
+      ChecklistSpec
+      Lib.External.AppManifestSpec
+      Lib.SoundSpec
+      Lib.Types.EmverProp
+      Live.Metrics
+      Live.Serialize
+      Spec
+  hs-source-dirs:
+      test
+  default-extensions:
+      NoImplicitPrelude
+      BlockArguments
+      ConstraintKinds
+      DataKinds
+      DeriveAnyClass
+      DeriveFunctor
+      DeriveGeneric
+      DerivingStrategies
+      EmptyCase
+      FlexibleContexts
+      FlexibleInstances
+      GADTs
+      GeneralizedNewtypeDeriving
+      InstanceSigs
+      KindSignatures
+      LambdaCase
+      MultiParamTypeClasses
+      MultiWayIf
+      NamedFieldPuns
+      NumericUnderscores
+      OverloadedStrings
+      PolyKinds
+      RankNTypes
+      StandaloneDeriving
+      StandaloneKindSignatures
+      TupleSections
+      TypeApplications
+      TypeFamilies
+      TypeOperators
+  ghc-options: -Wall -fdefer-typed-holes
+  build-depends:
+      aeson
+    , aeson-flatten
+    , ambassador-agent
+    , attoparsec
+    , base >=4.9.1.0 && <5
+    , bytestring
+    , casing
+    , comonad
+    , conduit
+    , conduit-extra
+    , connection
+    , containers
+    , cryptonite
+    , cryptonite-conduit
+    , data-default
+    , directory
+    , errors
+    , exceptions
+    , exinst
+    , fast-logger
+    , file-embed
+    , filelock
+    , filepath
+    , fused-effects
+    , fused-effects-th
+    , git-embed
+    , hedgehog
+    , hspec >=2.0.0
+    , hspec-expectations
+    , http-api-data
+    , http-client
+    , http-client-tls
+    , http-conduit
+    , http-types
+    , interpolate
+    , iso8601-time
+    , json-rpc
+    , lens
+    , lens-aeson
+    , lifted-async
+    , lifted-base
+    , memory
+    , mime-types
+    , monad-control
+    , monad-logger
+    , network
+    , persistent
+    , persistent-sqlite
+    , persistent-template
+    , process
+    , process-extras
+    , protolude
+    , random
+    , regex-compat
+    , resourcet
+    , shell-conduit
+    , singletons
+    , stm
+    , streaming
+    , streaming-bytestring
+    , streaming-conduit
+    , streaming-utils
+    , tar-conduit
+    , template-haskell
+    , text >=0.11 && <2.0
+    , time
+    , transformers
+    , transformers-base
+    , typed-process
+    , unix
+    , unliftio
+    , unliftio-core
+    , unordered-containers
+    , uuid
+    , wai
+    , wai-cors
+    , wai-extra
+    , warp
+    , yaml
+    , yesod
+    , yesod-auth
+    , yesod-core
+    , yesod-form
+    , yesod-persistent
+    , yesod-test
+  default-language: Haskell2010

agent/build.sh

@@ -3,4 +3,4 @@
 cat config/settings.yml | grep app-mgr-version-spec
 cat package.yaml | grep version
 
-stack --local-bin-path ./executables build --copy-bins #--flag start9-agent:disable-auth
+stack --local-bin-path ./dist build --copy-bins #--flag start9-agent:disable-auth

agent/cabal.project

@@ -0,0 +1,23 @@
+-- Generated by stackage-to-hackage
+
+index-state: 2021-04-26T18:08:38Z
+
+with-compiler: ghc-8.10.2
+
+packages:
+    ./
+
+source-repository-package
+    type: git
+    location: https://github.com/ProofOfKeags/persistent.git
+    tag: 3b52b13d9ce79cdef14bb1c37cc527657a529462
+    subdir: persistent-sqlite
+
+allow-older: *
+allow-newer: *
+
+package *
+    ghc-options: -haddock
+
+package ambassador-agent
+    ghc-options: -fwrite-ide-info

agent/config/restarter.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=restarts dead containers
+Requires=docker.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/appmgr repair-app-status

agent/config/restarter.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=restarter
+
+[Timer]
+OnUnitActiveSec=60s
+OnBootSec=60s
+
+[Install]
+WantedBy=timers.target

agent/config/routes

@@ -7,10 +7,13 @@
 /v0                                   ServerR                  GET PATCH
 
 /v0/name                              NameR                    PATCH
+/v0/autoCheckUpdates                  AutoCheckUpdatesR        PATCH
 
+/v0/welcome/#Version                  WelcomeR                 POST
 /v0/specs                             SpecsR                   GET
 /v0/metrics                           MetricsR                 GET
 
+/v0/logs                              LogsR                    GET
 /v0/sshKeys                           SshKeysR                 GET POST
 /v0/sshKeys/#Text                     SshKeyByFingerprintR     DELETE
 /v0/password                          PasswordR                PATCH
@@ -36,8 +39,12 @@
 /v0/apps/#AppId/backup/stop           StopBackupR              POST
 /v0/apps/#AppId/backup/restore        RestoreBackupR           POST
 /v0/apps/#AppId/autoconfig/#AppId     AutoconfigureR           POST
+/v0/apps/#AppId/actions               ActionR                  POST
 
-/v0/disks                             ListDisksR               GET
+/v0/network/lan/reset                 ResetLanR                POST
+
+/v0/disks                             DisksR                   GET
+/v0/disks/eject                       EjectR                   POST
 
 /v0/update                            UpdateAgentR             POST
 /v0/wifi                              WifiR                    GET POST

agent/config/settings.yml

@@ -1,9 +1,9 @@
 # Values formatted like "_env:YESOD_ENV_VAR_NAME:default_value" can be overridden by the specified environment variable.
 # See https://github.com/yesodweb/yesod/wiki/Configuration#overriding-configuration-values-with-environment-variables
 
-static-dir:     "_env:YESOD_STATIC_DIR:static"
+static-dir: "_env:YESOD_STATIC_DIR:static"
-host:           "_env:YESOD_HOST:*4" # any IPv4 host
+host: "_env:YESOD_HOST:*4" # any IPv4 host
-port:           5959 # NB: The port `yesod devel` uses is distinct from this value. Set the `yesod devel` port from the command line.
+port: 5959 # NB: The port `yesod devel` uses is distinct from this value. Set the `yesod devel` port from the command line.
 ip-from-header: "_env:YESOD_IP_FROM_HEADER:false"
 detailed-logging: "_env:DETAILED_LOGGING:false"
 
@@ -28,12 +28,10 @@ detailed-logging: "_env:DETAILED_LOGGING:false"
 
 # NB: If you need a numeric value (e.g. 123) to parse as a String, wrap it in single quotes (e.g. "_env:YESOD_PGPASS:'123'")
 # See https://github.com/yesodweb/yesod/wiki/Configuration#parsing-numeric-values-as-strings
-cors-override-star: "_env:CORS_OVERRIDE_STAR:"
 filesystem-base: "_env:FILESYSTEM_BASE:/"
 database:
   database: "start9_agent.sqlite3"
   poolsize: "_env:YESOD_SQLITE_POOLSIZE:10"
 
-app-mgr-version-spec: "=0.2.6"
+app-mgr-version-spec: "=0.2.16"
-
 #analytics: UA-YOURCODE

agent/migrations/0.2.10::0.2.11

@@ -0,0 +1 @@
+SELECT TRUE;

agent/migrations/0.2.11::0.2.12

@@ -0,0 +1 @@
+SELECT TRUE;

agent/migrations/0.2.12::0.2.13

@@ -0,0 +1 @@
+SELECT TRUE;

agent/migrations/0.2.13::0.2.14

@@ -0,0 +1 @@
+SELECT TRUE;

agent/migrations/0.2.14::0.2.15

@@ -0,0 +1 @@
+SELECT TRUE;

agent/migrations/0.2.15::0.2.16

@@ -0,0 +1 @@
+SELECT TRUE;

agent/migrations/0.2.6::0.2.7

@@ -0,0 +1 @@
+SELECT TRUE;

agent/migrations/0.2.7::0.2.8

@@ -0,0 +1 @@
+SELECT TRUE;

agent/migrations/0.2.8::0.2.9

@@ -0,0 +1 @@
+SELECT TRUE;

agent/migrations/0.2.9::0.2.10

@@ -0,0 +1 @@
+SELECT TRUE;

agent/package.yaml

@@ -1,114 +1,117 @@
 name: ambassador-agent
-version: 0.2.6
+version: 0.2.16
 
 default-extensions:
-- NoImplicitPrelude
+  - NoImplicitPrelude
-- BlockArguments
+  - BlockArguments
-- ConstraintKinds
+  - ConstraintKinds
-- DataKinds
+  - DataKinds
-- DeriveAnyClass
+  - DeriveAnyClass
-- DeriveFunctor
+  - DeriveFunctor
-- DeriveGeneric
+  - DeriveGeneric
-- DerivingStrategies
+  - DerivingStrategies
-- EmptyCase
+  - EmptyCase
-- FlexibleContexts
+  - FlexibleContexts
-- FlexibleInstances
+  - FlexibleInstances
-- GADTs
+  - GADTs
-- GeneralizedNewtypeDeriving
+  - GeneralizedNewtypeDeriving
-- InstanceSigs
+  - InstanceSigs
-- KindSignatures
+  - KindSignatures
-- LambdaCase
+  - LambdaCase
-- MultiParamTypeClasses
+  - MultiParamTypeClasses
-- MultiWayIf
+  - MultiWayIf
-- NamedFieldPuns
+  - NamedFieldPuns
-- NumericUnderscores
+  - NumericUnderscores
-- OverloadedStrings
+  - OverloadedStrings
-- PolyKinds
+  - PolyKinds
-- RankNTypes
+  - RankNTypes
-- StandaloneDeriving
+  - StandaloneDeriving
-- StandaloneKindSignatures
+  - StandaloneKindSignatures
-- TupleSections
+  - TupleSections
-- TypeApplications
+  - TypeApplications
-- TypeFamilies
+  - TypeFamilies
-- TypeOperators
+  - TypeOperators
 
 dependencies:
-- base >=4.9.1.0 && <5
+  - base >=4.9.1.0 && <5
-- aeson
+  - aeson
-- aeson-flatten
+  - aeson-flatten
-- attoparsec
+  - attoparsec
-- bytestring
+  - bytestring
-- casing
+  - casing
-- comonad
+  - comonad
-- conduit
+  - conduit
-- conduit-extra
+  - conduit-extra
-- containers
+  - connection
-- cryptonite
+  - containers
-- cryptonite-conduit
+  - cryptonite
-- data-default
+  - cryptonite-conduit
-- directory
+  - data-default
-- errors
+  - directory
-- exceptions
+  - errors
-- exinst
+  - exceptions
-- fast-logger
+  - exinst
-- file-embed
+  - fast-logger
-- filelock
+  - file-embed
-- filepath
+  - filelock
-- fused-effects
+  - filepath
-- fused-effects-th
+  - fused-effects
-- git-embed
+  - fused-effects-th
-- http-api-data
+  - git-embed
-- http-client
+  - http-api-data
-- http-client-tls
+  - http-client
-- http-conduit
+  - http-client-tls
-- http-types
+  - http-conduit
-- interpolate
+  - http-types
-- iso8601-time
+  - interpolate
-- lens
+  - iso8601-time
-- lens-aeson
+  - json-rpc
-- lifted-async
+  - lens
-- lifted-base
+  - lens-aeson
-- memory
+  - lifted-async
-- mime-types
+  - lifted-base
-- monad-control
+  - memory
-- monad-logger
+  - mime-types
-- persistent
+  - monad-control
-- persistent-sqlite
+  - monad-logger
-- persistent-template
+  - network
-- process
+  - persistent
-- process-extras
+  - persistent-sqlite
-- protolude
+  - persistent-template
-- resourcet
+  - process
-- regex-compat # TODO: trim this dep
+  - process-extras
-- shell-conduit
+  - protolude
-- singletons
+  - resourcet
-- stm
+  - regex-compat # TODO: trim this dep
-- streaming
+  - shell-conduit
-- streaming-bytestring
+  - singletons
-- streaming-conduit
+  - stm
-- streaming-utils
+  - streaming
-- tar-conduit
+  - streaming-bytestring
-- template-haskell
+  - streaming-conduit
-- text >=0.11 && <2.0
+  - streaming-utils
-- time
+  - tar-conduit
-- transformers
+  - template-haskell
-- transformers-base
+  - text >=0.11 && <2.0
-- typed-process
+  - time
-- unix
+  - transformers
-- unliftio # TODO: trim this dep
+  - transformers-base
-- unliftio-core # TODO: trim this dep
+  - typed-process
-- unordered-containers
+  - unix
-- uuid
+  - unliftio # TODO: trim this dep
-- wai
+  - unliftio-core # TODO: trim this dep
-- wai-cors
+  - unordered-containers
-- wai-extra
+  - uuid
-- warp
+  - wai
-- yaml
+  - wai-cors
-- yesod
+  - wai-extra
-- yesod-auth
+  - warp
-- yesod-core
+  - yaml
-- yesod-form
+  - yesod
-- yesod-persistent
+  - yesod-auth
+  - yesod-core
+  - yesod-form
+  - yesod-persistent
 
 flags:
   library-only:
@@ -126,56 +129,57 @@ flags:
 library:
   source-dirs: src
   when:
-  - condition: (flag(dev)) || (flag(library-only))
+    - condition: (flag(dev)) || (flag(library-only))
-    then:
+      then:
-      cpp-options: -DDEVELOPMENT
+        cpp-options: -DDEVELOPMENT
-      ghc-options:
+        ghc-options:
-      - -Wall
+          - -Wall
-      - -Wunused-packages
+          - -Wunused-packages
-      - -fwarn-tabs
+          - -fwarn-tabs
-      - -O0
+          - -O0
-      - -fdefer-typed-holes
+          - -fdefer-typed-holes
-    else:
+      else:
-      ghc-options:
+        ghc-options:
-      - -Wall
+          - -Wall
-      - -Wunused-packages
+          - -Wunused-packages
-      - -fwarn-tabs
+          - -fwarn-tabs
-      - -O2
+          - -O2
-      - -fdefer-typed-holes
+          - -fdefer-typed-holes
-  - condition: (flag(disable-auth))
+    - condition: (flag(disable-auth))
-    cpp-options: -DDISABLE_AUTH
+      cpp-options: -DDISABLE_AUTH
 tests:
   agent-test:
     source-dirs: test
     main: Main.hs
     ghc-options:
-    - -Wall
+      - -Wall
-    - -fdefer-typed-holes
+      - -fdefer-typed-holes
     dependencies:
-    - ambassador-agent
+      - ambassador-agent
-    - hspec >=2.0.0
+      - hspec >=2.0.0
-    - hspec-expectations
+      - hspec-expectations
-    - hedgehog
+      - hedgehog
-    - yesod-test
+      - yesod-test
-    - random
+      - random
     when:
-    - condition: false
+      - condition: false
-      other-modules: Paths_ambassador_agent
+        other-modules: Paths_ambassador_agent
 
 executables:
   agent:
     source-dirs: app
     main: main.hs
     ghc-options:
-    - -Wall
+      - -Wall
-    - -threaded
+      - -threaded
-    - -rtsopts
+      - -rtsopts
-    - -with-rtsopts=-N
+      - -with-rtsopts=-N
-    - -fdefer-typed-holes
+      - -fdefer-typed-holes
     dependencies:
-    - ambassador-agent
+      - ambassador-agent
     when:
-    - buildable: false
+      - buildable: false
-      condition: flag(library-only)
+        condition: flag(library-only)
-    - condition: false
+      - condition: false
-      other-modules: Paths_ambassador_agent
+        other-modules: Paths_ambassador_agent
+extra-source-files: ./migrations/*

agent/src/Application.hs

@@ -54,19 +54,21 @@ import           Yesod.Persist.Core
 import           Constants
 import qualified Daemon.AppNotifications       as AppNotifications
 import           Daemon.RefreshProcDev
+import qualified Daemon.SslRenew as SSLRenew
+import           Daemon.TorHealth
 import           Daemon.ZeroConf
 import           Foundation
 import           Lib.Algebra.State.RegistryUrl
+import           Lib.Background
 import           Lib.Database
 import           Lib.External.Metrics.ProcDev
 import           Lib.SelfUpdate
 import           Lib.Sound
 import           Lib.SystemPaths
+import           Lib.Tor                        ( newTorManager )
 import           Lib.WebServer
 import           Model
 import           Settings
-import Lib.Background
-import qualified Daemon.SslRenew as SSLRenew
 
 appMain :: IO ()
 appMain = do
@@ -106,13 +108,17 @@ makeFoundation appSettings = do
     -- subsite.
     appLogger                   <- newStdoutLoggerSet defaultBufSize >>= makeYesodLogger
     appHttpManager              <- getGlobalManager
+    appTorManager               <- newTorManager (appTorSocksPort appSettings)
     appWebServerThreadId        <- newIORef Nothing
     appSelfUpdateSpecification  <- newEmptyMVar
     appIsUpdating               <- newIORef Nothing
     appIsUpdateFailed           <- newIORef Nothing
+    appOsVersionLatest          <- newIORef Nothing
     appBackgroundJobs           <- newTVarIO (JobCache HM.empty)
     def                         <- getDefaultProcDevMetrics
     appProcDevMomentCache       <- newIORef (now, mempty, def)
+    appLastTorRestart           <- newIORef now
+    appLanThread                <- forkIO (sleep 10) >>= newMVar
 
     -- We need a log function to create a connection pool. We need a connection
     -- pool to create our foundation. And we need our foundation to get a
@@ -193,6 +199,10 @@ startupSequence foundation = do
         void . forkIO . forever $ forkIO (SSLRenew.renewSslLeafCert foundation) *> sleep 86_400
         withAgentVersionLog_ "SSL Renewal daemon started"
 
+        withAgentVersionLog_ "Initializing Tor health check loop"
+        void . forkIO . forever $ forkIO (runReaderT torHealth foundation) *> sleep 300
+        withAgentVersionLog_ "Tor health check loop running"
+
         -- reloading avahi daemon
         -- DRAGONS! make sure this step happens AFTER system synchronization
         withAgentVersionLog_ "Publishing Agent to Avahi Daemon"

agent/src/Daemon/TorHealth.hs

@@ -0,0 +1,50 @@
+{-# LANGUAGE QuasiQuotes #-}
+module Daemon.TorHealth where
+
+import           Startlude
+
+import           Data.String.Interpolate.IsString
+
+import           Foundation
+import           Lib.SystemPaths
+import           Lib.Tor
+import           Yesod                          ( RenderRoute(renderRoute) )
+import           Network.HTTP.Simple            ( getResponseBody )
+import           Network.HTTP.Client            ( parseRequest )
+import           Network.HTTP.Client            ( httpLbs )
+import           Data.ByteString.Lazy           ( toStrict )
+import qualified UnliftIO.Exception            as UnliftIO
+import           Settings
+import           Data.IORef                     ( writeIORef
+                                                , readIORef
+                                                )
+import           Lib.SystemCtl
+
+torHealth :: ReaderT AgentCtx IO ()
+torHealth = do
+    settings <- asks appSettings
+    host     <- injectFilesystemBaseFromContext settings getAgentHiddenServiceUrl
+    let url = mappend [i|http://#{host}:5959|] . fold $ mappend "/" <$> fst (renderRoute VersionR)
+    response <- UnliftIO.try @_ @SomeException $ torGet (toS url)
+    case response of
+        Left _ -> do
+            putStrLn @Text "Failed Tor health check"
+            lastRestart <- asks appLastTorRestart >>= liftIO . readIORef
+            cooldown    <- asks $ appTorRestartCooldown . appSettings
+            now         <- liftIO getCurrentTime
+            if now > addUTCTime cooldown lastRestart
+                then do
+                    ec <- liftIO $ systemCtl RestartService "tor"
+                    case ec of
+                        ExitSuccess   -> asks appLastTorRestart >>= liftIO . flip writeIORef now
+                        ExitFailure _ -> do
+                            putStrLn @Text "Failed to restart tor daemon after failed tor health check"
+                else do
+                    putStrLn @Text "Failed tor healthcheck inside of cooldown window, tor will not be restarted"
+        Right _ -> pure ()
+
+torGet :: String -> ReaderT AgentCtx IO ByteString
+torGet url = do
+    manager <- asks appTorManager
+    req     <- parseRequest url
+    liftIO $ toStrict . getResponseBody <$> httpLbs req manager

agent/src/Daemon/ZeroConf.hs

@@ -18,6 +18,9 @@ import           Lib.ProductKey
 import           Lib.SystemPaths
 
 import           Settings
+import qualified Lib.Algebra.Domain.AppMgr     as AppMgr2
+import           Control.Carrier.Lift
+import           Lib.Error
 
 start9AgentServicePrefix :: IsString a => a
 start9AgentServicePrefix = "start9-"
@@ -53,4 +56,10 @@ publishAgentToAvahi = do
         "_http._tcp"
         agentPort
     lift Avahi.reload
+    lift $ threadDelay 10_000_000
+    tid <- asks appLanThread >>= liftIO . takeMVar
+    liftIO $ killThread tid
+    tid' <- liftIO $ forkIO (runM . void . runExceptT @S9Error $ AppMgr2.runAppMgrCliC AppMgr2.lanEnable)
+    asks appLanThread >>= liftIO . flip putMVar tid'
+
 

agent/src/Foundation.hs

@@ -58,19 +58,24 @@ import           Settings
 -- keep settings and values requiring initialization before your application
 -- starts running, such as database connections. Every handler will have
 -- access to the data present here.
+data OsVersionCache = OsVersionCache { osVersion :: Version, lastChecked :: UTCTime }
 
 data AgentCtx = AgentCtx
     { appSettings                 :: AppSettings
     , appHttpManager              :: Manager
+    , appTorManager               :: Manager
     , appConnPool                 :: ConnectionPool -- ^ Database connection pool.
     , appLogger                   :: Logger
     , appWebServerThreadId        :: IORef (Maybe ThreadId)
     , appIsUpdating               :: IORef (Maybe Version)
     , appIsUpdateFailed           :: IORef (Maybe S9Error)
+    , appOsVersionLatest          :: IORef (Maybe OsVersionCache)
     , appProcDevMomentCache       :: IORef (UTCTime, ProcDevMomentStats, ProcDevMetrics)
     , appSelfUpdateSpecification  :: MVar VersionRange
     , appBackgroundJobs           :: TVar JobCache
     , appIconTags                 :: TVar (HM.HashMap AppId (Digest MD5))
+    , appLastTorRestart           :: IORef UTCTime
+    , appLanThread                :: MVar ThreadId
     }
 
 setWebProcessThreadId :: ThreadId -> AgentCtx -> IO ()
@@ -181,6 +186,8 @@ cutoffDuringUpdate m = do
             path <- asks $ pathInfo . reqWaiRequest . handlerRequest
             case path of
                 [v] | v == "v" <> (show . major $ agentVersion) -> m
+                [auth] | auth == "auth" -> m
+                (_:ssh:_) | ssh == "sshKeys" -> m
                 _ -> handleS9ErrT $ throwE UpdateInProgressE
         Nothing -> m
 

agent/src/Handler/Apps.hs

@@ -7,65 +7,70 @@
 {-# LANGUAGE TypeApplications #-}
 module Handler.Apps where
 
-import           Startlude               hiding ( modify
+import           Startlude               hiding ( Reader
-                                                , execState
                                                 , asks
-                                                , Reader
-                                                , runReader
                                                 , catchError
-                                                , forkFinally
                                                 , empty
+                                                , execState
+                                                , forkFinally
+                                                , modify
+                                                , runReader
                                                 )
 
-import           Control.Carrier.Reader
 import           Control.Carrier.Error.Church
 import           Control.Carrier.Lift
+import           Control.Carrier.Reader
 import qualified Control.Concurrent.Async.Lifted
                                                as LAsync
 import qualified Control.Concurrent.Lifted     as Lifted
-import qualified Control.Exception.Lifted      as Lifted
 import           Control.Concurrent.STM.TVar
 import           Control.Effect.Empty    hiding ( guard )
 import           Control.Effect.Labelled        ( HasLabelled
                                                 , Labelled
                                                 , runLabelled
                                                 )
+import qualified Control.Exception.Lifted      as Lifted
 import           Control.Lens            hiding ( (??) )
 import           Control.Monad.Logger
 import           Control.Monad.Trans.Control    ( MonadBaseControl )
+import           Crypto.Hash
 import           Data.Aeson
 import           Data.Aeson.Lens
+import           Data.Aeson.Types               ( parseMaybe )
 import qualified Data.ByteString.Lazy          as LBS
-import           Data.IORef
 import qualified Data.HashMap.Lazy             as HML
 import qualified Data.HashMap.Strict           as HM
+import           Data.IORef
 import qualified Data.List.NonEmpty            as NE
 import           Data.Singletons
-import           Data.Singletons.Prelude.Bool   ( SBool(..)
+import           Data.Singletons.Prelude.Bool   ( If
-                                                , If
+                                                , SBool(..)
                                                 )
 import           Data.Singletons.Prelude.List   ( Elem )
-
+import qualified Data.Text                     as Text
 import           Database.Persist
 import           Database.Persist.Sql           ( ConnectionPool )
 import           Database.Persist.Sqlite        ( runSqlPool )
 import           Exinst
 import           Network.HTTP.Types
+import qualified Network.JSONRPC               as JSONRPC
 import           Yesod.Core.Content
-import           Yesod.Core.Json
 import           Yesod.Core.Handler      hiding ( cached )
+import           Yesod.Core.Json
 import           Yesod.Core.Types               ( JSONResponse(..) )
 import           Yesod.Persist.Core
 
 import           Foundation
 import           Handler.Backups
 import           Handler.Icons
+import           Handler.Network
 import           Handler.Types.Apps
 import           Handler.Util
 import qualified Lib.Algebra.Domain.AppMgr     as AppMgr2
 import           Lib.Algebra.State.RegistryUrl
 import           Lib.Background
 import           Lib.Error
+import qualified Lib.External.AppManifest      as AppManifest
 import qualified Lib.External.AppMgr           as AppMgr
 import qualified Lib.External.Registry         as Reg
 import           Lib.IconCache
@@ -74,10 +79,10 @@ import           Lib.SystemPaths
 import           Lib.TyFam.ConditionalData
 import           Lib.Types.Core
 import           Lib.Types.Emver
+import           Lib.Types.NetAddress
 import           Lib.Types.ServerApp
 import           Model
 import           Settings
-import           Crypto.Hash
 
 pureLog :: Show a => a -> Handler a
 pureLog = liftA2 (*>) ($logInfo . show) pure
@@ -104,7 +109,11 @@ type AllEffects m
                             ( Labelled
                                   "databaseConnection"
                                   (ReaderT ConnectionPool)
-                                  (ReaderT AgentCtx (ErrorC S9Error (LiftC m)))
+                                  ( Labelled
+                                        "lanThread"
+                                        (ReaderT (MVar ThreadId))
+                                        (ReaderT AgentCtx (ErrorC S9Error (LiftC m)))
+                                  )
                             )
                       )
                 )
@@ -117,6 +126,8 @@ intoHandler m = do
     runM
         . handleS9ErrC
         . flip runReaderT ctx
+        . flip runReaderT (appLanThread ctx)
+        . runLabelled @"lanThread"
         . flip runReaderT (appConnPool ctx)
         . runLabelled @"databaseConnection"
         . flip runReaderT fsbase
@@ -143,8 +154,7 @@ getAvailableAppsLogic :: ( Has (Reader AgentCtx) sig m
 getAvailableAppsLogic = do
     jobCache <- asks appBackgroundJobs >>= liftIO . readTVarIO
     let installCache = inspect SInstalling jobCache
-    (Reg.AppManifestRes apps, serverApps) <- LAsync.concurrently Reg.getAppManifest
+    (Reg.AppIndexRes apps, serverApps) <- LAsync.concurrently Reg.getAppIndex (AppMgr2.list [AppMgr2.flags|-s -d|])
-                                                                 (AppMgr2.list [AppMgr2.flags|-s -d|])
     let remapped = remapAppMgrInfo jobCache serverApps
     pure $ foreach apps $ \app@StoreApp { storeAppId } ->
         let installing =
@@ -172,8 +182,9 @@ getAvailableAppByIdLogic appId = do
     let storeAppId' = storeAppId
     jobCache <- asks appBackgroundJobs >>= liftIO . readTVarIO
     let installCache = inspect SInstalling jobCache
-    (Reg.AppManifestRes storeApps, serverApps) <- LAsync.concurrently Reg.getAppManifest
+    ((Reg.AppIndexRes storeApps, serverApps), AppManifest.AppManifest { appManifestLicenseName, appManifestLicenseLink }) <-
-                                                                      (AppMgr2.list [AppMgr2.flags|-s -d|])
+        LAsync.concurrently (LAsync.concurrently Reg.getAppIndex (AppMgr2.list [AppMgr2.flags|-s -d|]))
+                            (Reg.getAppManifest appId)
     StoreApp {..} <- pure (find ((== appId) . storeAppId) storeApps) `orThrowM` NotFoundE "appId" (show appId)
     let remapped = remapAppMgrInfo jobCache serverApps
     let installingInfo =
@@ -202,6 +213,8 @@ getAvailableAppByIdLogic appId = do
                                                        appId
                                                        storeAppTitle
                                                        (storeIconUrl appId (storeAppVersionInfoVersion $ extract storeAppVersions))
+        , appAvailableFullLicenseName            = appManifestLicenseName
+        , appAvailableFullLicenseLink            = appManifestLicenseLink
         , appAvailableFullInstallInfo            = installingInfo
         , appAvailableFullVersionLatest          = storeAppVersionInfoVersion latest
         , appAvailableFullDescriptionShort       = storeAppDescriptionShort
@@ -209,6 +222,7 @@ getAvailableAppByIdLogic appId = do
         , appAvailableFullReleaseNotes           = storeAppVersionInfoReleaseNotes latest
         , appAvailableFullDependencyRequirements = HM.elems dependencyRequirements
         , appAvailableFullVersions               = storeAppVersionInfoVersion <$> storeAppVersions
+        , appAvailableFullInstallAlert           = storeAppVersionInfoInstallAlert latest
         }
 
 getAppLogsByIdR :: AppId -> Handler (JSONResponse [Text])
@@ -230,7 +244,7 @@ getInstalledAppsLogic :: (Has (Reader AgentCtx) sig m, Has AppMgr2.AppMgr sig m,
 getInstalledAppsLogic = do
     jobCache <- asks appBackgroundJobs >>= liftIO . readTVarIO
     let installCache = installInfo . fst <$> inspect SInstalling jobCache
-    serverApps <- AppMgr2.list [AppMgr2.flags|-s -d|]
+    serverApps <- AppMgr2.list [AppMgr2.flags|-s -d -m|]
     let remapped           = remapAppMgrInfo jobCache serverApps
         installingPreviews = flip
             HM.mapWithKey
@@ -242,16 +256,30 @@ getInstalledAppsLogic = do
                 , appInstalledPreviewStatus           = AppStatusTmp Installing
                 , appInstalledPreviewVersionInstalled = storeAppVersionInfoVersion
                 , appInstalledPreviewTorAddress       = Nothing
+                , appInstalledPreviewLanAddress       = Nothing
+                , appInstalledPreviewTorUi            = False
+                , appInstalledPreviewLanUi            = False
                 }
         installedPreviews = flip
             HML.mapWithKey
             remapped
-            \appId (s, v, AppMgr2.InfoRes {..}) -> AppInstalledPreview
+            \appId (s, v, AppMgr2.InfoRes {..}) ->
-                { appInstalledPreviewBase             = AppBase appId infoResTitle (iconUrl appId v)
+                let
-                , appInstalledPreviewStatus           = s
+                    mLanAddress = do -- Maybe
-                , appInstalledPreviewVersionInstalled = v
+                        addrBase <- infoResTorAddress
-                , appInstalledPreviewTorAddress       = infoResTorAddress
+                        let
-                }
+                            lanConfs = mapMaybe AppManifest.portMapEntryLan
+                                $ AppManifest.appManifestPortMapping infoResManifest
+                        guard (not . null $ lanConfs)
+                        pure $ LanAddress . (".onion" `Text.replace` ".local") . unTorAddress $ addrBase
+                in  AppInstalledPreview { appInstalledPreviewBase = AppBase appId infoResTitle (iconUrl appId v)
+                                        , appInstalledPreviewStatus = s
+                                        , appInstalledPreviewVersionInstalled = v
+                                        , appInstalledPreviewTorAddress = infoResTorAddress
+                                        , appInstalledPreviewLanAddress = mLanAddress
+                                        , appInstalledPreviewTorUi = AppManifest.torUiAvailable infoResManifest
+                                        , appInstalledPreviewLanUi = AppManifest.lanUiAvailable infoResManifest
+                                        }
 
     pure $ HML.elems $ HML.union installingPreviews installedPreviews
 
@@ -277,19 +305,29 @@ getInstalledAppByIdLogic appId = do
             backupTime <- lift $ LAsync.wait backupTime'
             hoistMaybe $ HM.lookup appId installCache <&> \(StoreApp {..}, StoreAppVersionInfo {..}) -> AppInstalledFull
                 { appInstalledFullBase = AppBase appId storeAppTitle (iconUrl appId storeAppVersionInfoVersion)
+                , appInstalledFullLicenseName            = Nothing
+                , appInstalledFullLicenseLink            = Nothing
                 , appInstalledFullStatus                 = AppStatusTmp Installing
                 , appInstalledFullVersionInstalled       = storeAppVersionInfoVersion
                 , appInstalledFullInstructions           = Nothing
                 , appInstalledFullLastBackup             = backupTime
                 , appInstalledFullTorAddress             = Nothing
+                , appInstalledFullLanAddress             = Nothing
+                , appInstalledFullTorUi                  = False
+                , appInstalledFullLanUi                  = False
                 , appInstalledFullConfiguredRequirements = []
+                , appInstalledFullUninstallAlert         = Nothing
+                , appInstalledFullRestoreAlert           = Nothing
+                , appInstalledFullStartAlert             = Nothing
+                , appInstalledFullActions                = []
                 }
     serverApps <- AppMgr2.list [AppMgr2.flags|-s -d|]
     let remapped = remapAppMgrInfo jobCache serverApps
-    appManifestFetchCached <- cached Reg.getAppManifest
+    appManifestFetchCached <- cached Reg.getAppIndex
     let
         installed = do
             (status, version, AppMgr2.InfoRes {..}) <- hoistMaybe (HM.lookup appId remapped)
+            manifest' <- lift $ LAsync.async $ AppMgr2.infoResManifest <<$>> AppMgr2.info [AppMgr2.flags|-M|] appId
             instructions'                           <- lift $ LAsync.async $ AppMgr2.instructions appId
             requirements                            <- LAsync.runConcurrently $ flip
                 HML.traverseWithKey
@@ -299,7 +337,7 @@ getInstalledAppByIdLogic appId = do
                         fromInstalled = (AppMgr2.infoResTitle &&& AppMgr2.infoResVersion)
                             <$> hoistMaybe (HM.lookup depId serverApps)
                     let fromStore = do
-                            Reg.AppManifestRes res <- lift appManifestFetchCached
+                            Reg.AppIndexRes res <- lift appManifestFetchCached
                             (storeAppTitle &&& storeAppVersionInfoVersion . extract . storeAppVersions)
                                 <$> hoistMaybe (find ((== depId) . storeAppId) res)
                     (title, v) <- fromInstalled <|> fromStore
@@ -309,15 +347,32 @@ getInstalledAppByIdLogic appId = do
                             (HM.lookup depId installCache $> AppStatusTmp Installing)
                                 <|> (view _1 <$> HM.lookup depId remapped)
                     pure $ dependencyInfoToDependencyRequirement (AsInstalled STrue) (base, depStatus, depInfo)
+            manifest <- (lift $ LAsync.wait manifest') >>= \case
+                Nothing -> throwError $ NotFoundE "manifest" (show appId)
+                Just x  -> pure x
             instructions <- lift $ LAsync.wait instructions'
             backupTime   <- lift $ LAsync.wait backupTime'
+            let lanAddress = do
+                    addrBase <- infoResTorAddress
+                    let lanConfs = mapMaybe AppManifest.portMapEntryLan $ AppManifest.appManifestPortMapping manifest
+                    guard (not . null $ lanConfs)
+                    pure $ LanAddress . (".onion" `Text.replace` ".local") . unTorAddress $ addrBase
             pure AppInstalledFull { appInstalledFullBase = AppBase appId infoResTitle (iconUrl appId version)
+                                  , appInstalledFullLicenseName            = AppManifest.appManifestLicenseName manifest
+                                  , appInstalledFullLicenseLink            = AppManifest.appManifestLicenseLink manifest
                                   , appInstalledFullStatus                 = status
                                   , appInstalledFullVersionInstalled       = version
                                   , appInstalledFullInstructions           = instructions
                                   , appInstalledFullLastBackup             = backupTime
                                   , appInstalledFullTorAddress             = infoResTorAddress
+                                  , appInstalledFullLanAddress             = lanAddress
+                                  , appInstalledFullTorUi                  = AppManifest.torUiAvailable manifest
+                                  , appInstalledFullLanUi                  = AppManifest.lanUiAvailable manifest
                                   , appInstalledFullConfiguredRequirements = HM.elems requirements
+                                  , appInstalledFullUninstallAlert = AppManifest.appManifestUninstallAlert manifest
+                                  , appInstalledFullRestoreAlert = AppManifest.appManifestRestoreAlert manifest
+                                  , appInstalledFullStartAlert             = AppManifest.appManifestStartAlert manifest
+                                  , appInstalledFullActions                = AppManifest.appManifestActions manifest
                                   }
     runMaybeT (installing <|> installed) `orThrowM` NotFoundE "appId" (show appId)
 
@@ -333,6 +388,7 @@ postUninstallAppLogic :: ( HasFilesystemBase sig m
                          , MonadIO m
                          , HasLabelled "databaseConnection" (Reader ConnectionPool) sig m
                          , HasLabelled "iconTagCache" (Reader (TVar (HM.HashMap AppId (Digest MD5)))) sig m
+                         , HasLabelled "lanThread" (Reader (MVar ThreadId)) sig m
                          )
                       => AppId
                       -> AppMgr2.DryRun
@@ -351,7 +407,9 @@ postUninstallAppLogic appId dryrun = do
     breakageIds <- HM.keys . AppMgr2.unBreakageMap <$> AppMgr2.remove flags appId
     bs <- pure (traverse (hydrate $ (AppMgr2.infoResTitle &&& AppMgr2.infoResVersion) <$> serverApps) breakageIds)
         `orThrowM` InternalE "Reported app breakage for app that isn't installed, contact support"
-    when (not $ coerce dryrun) $ clearIcon appId
+    when (not $ coerce dryrun) $ do
+        clearIcon appId
+        postResetLanLogic
     pure $ WithBreakages bs ()
 
 type InstallResponse :: Bool -> Type
@@ -368,6 +426,7 @@ postInstallNewAppR appId = do
 
 postInstallNewAppLogic :: forall sig m a
                         . ( Has (Reader AgentCtx) sig m
+                          , HasLabelled "lanThread" (Reader (MVar ThreadId)) sig m
                           , HasLabelled "databaseConnection" (Reader ConnectionPool) sig m
                           , HasLabelled "iconTagCache" (Reader (TVar (HM.HashMap AppId (Digest MD5)))) sig m
                           , Has (Error S9Error) sig m
@@ -455,6 +514,7 @@ postInstallNewAppLogic appId appVersion dryrun = do
                                  (void $ Notifications.emit k infoResVersion (Notifications.RestartFailed e))
                                  pool
                 )
+            postResetLanLogic
 
 
 postStartServerAppR :: AppId -> Handler ()
@@ -620,8 +680,8 @@ getAvailableAppVersionInfoLogic :: ( Has (Reader AgentCtx) sig m
                                 -> VersionRange
                                 -> m AppVersionInfo
 getAvailableAppVersionInfoLogic appId appVersionSpec = do
-    jobCache                     <- asks appBackgroundJobs >>= liftIO . readTVarIO
+    jobCache                  <- asks appBackgroundJobs >>= liftIO . readTVarIO
-    Reg.AppManifestRes storeApps <- Reg.getAppManifest
+    Reg.AppIndexRes storeApps <- Reg.getAppIndex
     let titles =
             (storeAppTitle &&& storeAppVersionInfoVersion . extract . storeAppVersions) <$> indexBy storeAppId storeApps
     StoreApp {..} <- find ((== appId) . storeAppId) storeApps `orThrowPure` NotFoundE "appId" (show appId)
@@ -642,6 +702,7 @@ getAvailableAppVersionInfoLogic appId appVersionSpec = do
     pure AppVersionInfo { appVersionInfoVersion                = storeAppVersionInfoVersion
                         , appVersionInfoReleaseNotes           = storeAppVersionInfoReleaseNotes
                         , appVersionInfoDependencyRequirements = HM.elems requirements
+                        , appVersionInfoInstallAlert           = storeAppVersionInfoInstallAlert
                         }
 
 postAutoconfigureR :: AppId -> AppId -> Handler (JSONResponse (WithBreakages AutoconfigureChangesRes))
@@ -710,7 +771,6 @@ remapAppMgrInfo jobCache serverApps = flip
                     $   ((, infoResVersion) <$> HM.lookup appId tmpStatuses)
                     <|> (guard (not infoResIsConfigured || infoResIsRecoverable) $> (NeedsConfig, infoResVersion))
                     <|> (guard realViolations $> (BrokenDependencies, infoResVersion))
-                    <|> (guard (infoResStatus == Restarting) $> (Crashed, infoResVersion))
         in  ( status
             , version
             , infoRes
@@ -731,6 +791,7 @@ storeAppToAvailablePreview s@StoreApp {..} installed = AppAvailablePreview
     (storeAppVersionInfoVersion $ extract storeAppVersions)
     storeAppDescriptionShort
     installed
+    storeAppTimestamp
 
 type AsInstalled :: Bool -> Type
 newtype AsInstalled a = AsInstalled { unAsInstalled :: SBool a }
@@ -758,3 +819,21 @@ dependencyInfoToDependencyRequirement asInstalled (base, status, AppMgr2.Depende
             let appDependencyRequirementReasonOptional = dependencyInfoReasonOptional
                 appDependencyRequirementDefault        = dependencyInfoRequired
             in  AppDependencyRequirement { .. }
+
+postActionR :: AppId -> Handler (JSONResponse JSONRPC.Response)
+postActionR appId = do
+    req <- requireCheckJsonBody
+    fmap JSONResponse . intoHandler $ postActionLogic appId req
+
+postActionLogic :: (Has (Error S9Error) sig m, Has AppMgr2.AppMgr sig m)
+                => AppId
+                -> JSONRPC.Request
+                -> m JSONRPC.Response
+postActionLogic appId (JSONRPC.Request { getReqMethod, getReqId }) = do
+    hm <- AppMgr2.action appId getReqMethod
+    case (HM.lookup "result" hm, HM.lookup "error" hm >>= parseMaybe parseJSON) of
+        (Just v , _      ) -> pure (JSONRPC.Response JSONRPC.V2 v getReqId)
+        (_      , Just e ) -> pure (JSONRPC.ResponseError JSONRPC.V2 e getReqId)
+        (Nothing, Nothing) -> throwError
+            $ AppMgrParseE "action" (decodeUtf8 . LBS.toStrict $ encode (Object hm)) "Invalid JSONRPC Response"
+postActionLogic _ r = throwError $ InvalidRequestE (toJSON r) "Invalid JSONRPC Request"

agent/src/Handler/Backups.hs

@@ -7,11 +7,11 @@ import           Startlude               hiding ( Reader
                                                 , runReader
                                                 )
 
-import           Control.Effect.Labelled hiding ( Handler )
-import           Control.Effect.Reader.Labelled
 import           Control.Carrier.Error.Church
 import           Control.Carrier.Lift
 import           Control.Carrier.Reader         ( runReader )
+import           Control.Effect.Labelled hiding ( Handler )
+import           Control.Effect.Reader.Labelled
 import           Data.Aeson
 import qualified Data.HashMap.Strict           as HM
 import           Data.UUID.V4
@@ -20,8 +20,13 @@ import           Yesod.Auth
 import           Yesod.Core
 import           Yesod.Core.Types
 
+import           Control.Concurrent.STM
+import           Exinst
 import           Foundation
+import           Handler.Network
 import           Handler.Util
+import qualified Lib.Algebra.Domain.AppMgr     as AppMgr2
+import           Lib.Background
 import           Lib.Error
 import qualified Lib.External.AppMgr           as AppMgr
 import qualified Lib.Notifications             as Notifications
@@ -29,10 +34,6 @@ import           Lib.Password
 import           Lib.Types.Core
 import           Lib.Types.Emver
 import           Model
-import qualified Lib.Algebra.Domain.AppMgr     as AppMgr2
-import           Lib.Background
-import           Control.Concurrent.STM
-import           Exinst
 
 
 data CreateBackupReq = CreateBackupReq
@@ -57,6 +58,15 @@ instance FromJSON RestoreBackupReq where
         restoreBackupPassword    <- o .:? "password" .!= Nothing
         pure RestoreBackupReq { .. }
 
+data EjectDiskReq = EjectDiskReq
+    { ejectDiskLogicalName :: Text
+    }
+    deriving (Eq, Show)
+instance FromJSON EjectDiskReq where
+    parseJSON = withObject "Eject Disk Req" $ \o -> do
+        ejectDiskLogicalName <- o .: "logicalName"
+        pure EjectDiskReq { .. }
+
 -- Handlers
 
 postCreateBackupR :: AppId -> Handler ()
@@ -92,12 +102,16 @@ postRestoreBackupR appId = disableEndpointOnFailedUpdate $ do
         & runReader appConnPool
         & runLabelled @"backgroundJobCache"
         & runReader appBackgroundJobs
+        & runLabelled @"lanThread"
+        & runReader appLanThread
         & handleS9ErrC
         & runM
 
-getListDisksR :: Handler (JSONResponse [AppMgr.DiskInfo])
+getDisksR :: Handler (JSONResponse [AppMgr.DiskInfo])
-getListDisksR = fmap JSONResponse . runM . handleS9ErrC $ listDisksLogic
+getDisksR = fmap JSONResponse . runM . handleS9ErrC $ listDisksLogic
 
+postEjectR :: Handler ()
+postEjectR = runM . handleS9ErrC $ requireCheckJsonBody >>= ejectDiskLogic . ejectDiskLogicalName
 
 -- Logic
 
@@ -163,6 +177,7 @@ stopBackupLogic appId = do
 
 restoreBackupLogic :: ( HasLabelled "backgroundJobCache" (Reader (TVar JobCache)) sig m
                       , HasLabelled "databaseConnection" (Reader ConnectionPool) sig m
+                      , HasLabelled "lanThread" (Reader (MVar ThreadId)) sig m
                       , Has (Error S9Error) sig m
                       , Has AppMgr2.AppMgr sig m
                       , MonadIO m
@@ -171,10 +186,11 @@ restoreBackupLogic :: ( HasLabelled "backgroundJobCache" (Reader (TVar JobCache)
                    -> RestoreBackupReq
                    -> m ()
 restoreBackupLogic appId RestoreBackupReq {..} = do
-    jobCache <- ask @"backgroundJobCache"
+    lanThread <- ask @"lanThread"
-    db       <- ask @"databaseConnection"
+    jobCache  <- ask @"backgroundJobCache"
-    version  <- fmap AppMgr2.infoResVersion $ AppMgr2.info [AppMgr2.flags| |] appId `orThrowM` NotFoundE "appId"
+    db        <- ask @"databaseConnection"
-                                                                                                         (show appId)
+    version   <- fmap AppMgr2.infoResVersion $ AppMgr2.info [AppMgr2.flags| |] appId `orThrowM` NotFoundE "appId"
+                                                                                                          (show appId)
     res <- liftIO . atomically $ do
         (JobCache jobs) <- readTVar jobCache
         case HM.lookup appId jobs of
@@ -196,13 +212,23 @@ restoreBackupLogic appId RestoreBackupReq {..} = do
                 let notif = case appmgrRes of
                         Left  e -> Notifications.RestoreFailed e
                         Right _ -> Notifications.RestoreSucceeded
+                resetRes <- runExceptT @S9Error $ runReader lanThread . runLabelled @"lanThread" $ postResetLanLogic
+                case resetRes of
+                    Left  _  -> pure () -- temporarily forbidden is the only possible thing here so ignore it
+                    Right () -> pure ()
                 flip runSqlPool db $ void $ Notifications.emit appId version notif
             liftIO . atomically $ modifyTVar jobCache (insertJob appId Restore tid)
 
-
 listDisksLogic :: (Has (Error S9Error) sig m, MonadIO m) => m [AppMgr.DiskInfo]
 listDisksLogic = runExceptT AppMgr.diskShow >>= liftEither
 
+ejectDiskLogic :: (Has (Error S9Error) sig m, MonadIO m) => Text -> m ()
+ejectDiskLogic t = do
+    (ec, _) <- AppMgr.readProcessInheritStderr "eject" [toS t] ""
+    case ec of
+        ExitSuccess   -> pure ()
+        ExitFailure n -> throwError $ EjectE n
+
 insertBackupResult :: MonadIO m => AppId -> Version -> Bool -> SqlPersistT m (Entity BackupRecord)
 insertBackupResult appId appVersion succeeded = do
     uuid <- liftIO nextRandom

agent/src/Handler/Hosts.hs

@@ -5,26 +5,23 @@ module Handler.Hosts where
 import           Startlude               hiding ( ask )
 
 import           Control.Carrier.Lift           ( runM )
-import           Control.Carrier.Error.Church
 import           Data.Conduit
 import qualified Data.Conduit.Binary           as CB
-import           Data.Time.ISO8601
 import           Yesod.Core              hiding ( expiresAt )
 
 import           Foundation
-import           Daemon.ZeroConf
+import           Handler.Register               ( checkExistingPasswordRegistration
-import           Handler.Register               ( produceProofOfKey
+                                                , getRegistration
-                                                , checkExistingPasswordRegistration
                                                 )
 import           Handler.Types.Hosts
-import           Handler.Types.Register
 import           Lib.Crypto
 import           Lib.Error
 import           Lib.Password                   ( rootAccountName )
 import           Lib.ProductKey
-import           Lib.Ssl
+import           Lib.SystemPaths                ( injectFilesystemBaseFromContext
-import           Lib.SystemPaths
+                                                , rootCaCertPath
-import           Lib.Tor
+                                                , SystemPath(relativeTo)
+                                                )
 import           Settings
 
 getHostsR :: Handler HostsRes
@@ -34,7 +31,6 @@ getHostsR = handleS9ErrT $ do
     hostParams <- extractHostsQueryParams
 
     verifyHmac productKey hostParams
-    verifyTimestampNotExpired $ hostsParamsExpiration hostParams
 
     mClaimedAt <- checkExistingPasswordRegistration rootAccountName
     case mClaimedAt of
@@ -52,31 +48,6 @@ verifyHmac productKey params = do
         HostsParams { hostsParamsHmac, hostsParamsExpiration, hostsParamsSalt } = params
         unauthorizedHmac = ClientCryptographyE "Unauthorized hmac"
 
-verifyTimestampNotExpired :: MonadIO m => Text -> S9ErrT m ()
-verifyTimestampNotExpired expirationTimestamp = do
-    now <- liftIO getCurrentTime
-    case parseISO8601 . toS $ expirationTimestamp of
-        Nothing         -> throwE $ TTLExpirationE "invalid timestamp"
-        Just expiration -> when (expiration < now) (throwE $ TTLExpirationE "expired")
-
-getRegistration :: (MonadIO m, HasFilesystemBase sig m, Has (Error S9Error) sig m) => Text -> UTCTime -> m RegisterRes
-getRegistration productKey registerResClaimedAt = do
-    torAddress <- getAgentHiddenServiceUrlMaybe >>= \case
-        Nothing -> throwError $ NotFoundE "prior registration" "torAddress"
-        Just t  -> pure $ t
-    caCert <- readSystemPath rootCaCertPath >>= \case
-        Nothing -> throwError $ NotFoundE "prior registration" "cert"
-        Just t  -> pure t
-
-    -- create an hmac of the torAddress + caCert for front end
-    registerResTorAddressSig <- produceProofOfKey productKey torAddress
-    registerResCertSig       <- produceProofOfKey productKey caCert
-
-    let registerResCertName = root_CA_CERT_NAME
-    registerResLanAddress <- getStart9AgentHostnameLocal
-
-    pure RegisterRes { .. }
-
 getCertificateR :: Handler TypedContent
 getCertificateR = do
     base <- getsYesod $ appFilesystemBase . appSettings

agent/src/Handler/Icons.hs

@@ -14,6 +14,13 @@ import           Network.HTTP.Simple
 import           System.FilePath.Posix
 import           Yesod.Core
 
+import           Control.Carrier.Reader  hiding ( asks )
+import           Control.Concurrent.STM         ( modifyTVar
+                                                , readTVarIO
+                                                )
+import           Control.Effect.Labelled        ( runLabelled )
+import           Crypto.Hash.Conduit            ( hashFile )
+import qualified Data.HashMap.Strict           as HM
 import           Foundation
 import           Lib.Algebra.State.RegistryUrl
 import           Lib.Error
@@ -21,16 +28,9 @@ import qualified Lib.External.Registry         as Reg
 import           Lib.IconCache
 import           Lib.SystemPaths         hiding ( (</>) )
 import           Lib.Types.Core
+import           Lib.Types.Emver
 import           Lib.Types.ServerApp
 import           Settings
-import           Control.Carrier.Reader  hiding ( asks )
-import           Control.Effect.Labelled        ( runLabelled )
-import qualified Data.HashMap.Strict           as HM
-import           Control.Concurrent.STM         ( modifyTVar
-                                                , readTVarIO
-                                                )
-import           Crypto.Hash.Conduit            ( hashFile )
-import           Lib.Types.Emver
 
 iconUrl :: AppId -> Version -> Text
 iconUrl appId version = (foldMap (T.cons '/') . fst . renderRoute . AppIconR $ appId) <> "?" <> show version
@@ -63,7 +63,7 @@ getAppIconR appId = handleS9ErrT $ do
     lift $ respondSource (parseContentType path) $ CB.sourceFile path .| awaitForever sendChunkBS
     where
         fetchIcon = do
-            url <- find ((== appId) . storeAppId) . Reg.storeApps <$> Reg.getAppManifest >>= \case
+            url <- find ((== appId) . storeAppId) . Reg.storeApps <$> Reg.getAppIndex >>= \case
                 Nothing -> throwError $ NotFoundE "icon" (show appId)
                 Just x  -> pure . toS $ storeAppIconUrl x
             bp <- getAbsoluteLocationFor iconBasePath
@@ -84,7 +84,7 @@ getAvailableAppIconR :: AppId -> Handler TypedContent
 getAvailableAppIconR appId = handleS9ErrT $ do
     s   <- getsYesod appSettings
     url <- do
-        find ((== appId) . storeAppId) . Reg.storeApps <$> interp s Reg.getAppManifest >>= \case
+        find ((== appId) . storeAppId) . Reg.storeApps <$> interp s Reg.getAppIndex >>= \case
             Nothing -> throwE $ NotFoundE "icon" (show appId)
             Just x  -> pure . toS $ storeAppIconUrl x
     req <- case parseRequest url of

agent/src/Handler/Network.hs

@@ -0,0 +1,36 @@
+module Handler.Network where
+
+import           Startlude               hiding ( Reader
+                                                , ask
+                                                , asks
+                                                , runReader
+                                                )
+
+import           Control.Carrier.Lift           ( runM )
+import           Control.Effect.Error
+import           Lib.Error
+import           Yesod.Core                     ( getYesod )
+
+import           Control.Carrier.Reader         ( runReader )
+import           Control.Effect.Labelled        ( runLabelled )
+import           Control.Effect.Reader.Labelled
+import           Foundation
+import qualified Lib.Algebra.Domain.AppMgr     as AppMgr2
+import           Lib.Types.Core
+
+postResetLanR :: Handler ()
+postResetLanR = do
+    ctx <- getYesod
+    runM . handleS9ErrC . runReader (appLanThread ctx) . runLabelled @"lanThread" $ postResetLanLogic
+
+postResetLanLogic :: (MonadIO m, HasLabelled "lanThread" (Reader (MVar ThreadId)) sig m, Has (Error S9Error) sig m)
+                  => m ()
+postResetLanLogic = do
+    threadVar <- ask @"lanThread"
+    mtid      <- liftIO . tryTakeMVar $ threadVar
+    case mtid of
+        Nothing  -> throwError $ TemporarilyForbiddenE (AppId "LAN") "reset" "being reset"
+        Just tid -> liftIO $ do
+            killThread tid
+            newTid <- forkIO (void . runM . runExceptT @S9Error . AppMgr2.runAppMgrCliC $ AppMgr2.lanEnable)
+            putMVar threadVar newTid

agent/src/Handler/Register.hs

@@ -5,7 +5,10 @@ module Handler.Register where
 
 import           Startlude               hiding ( ask )
 
-import           Control.Carrier.Error.Either   ( runError )
+import           Control.Carrier.Error.Either   ( runError
+                                                , Error
+                                                , throwError
+                                                )
 import           Control.Carrier.Lift
 import           Control.Effect.Throw           ( liftEither )
 import           Crypto.Cipher.Types
@@ -29,6 +32,7 @@ import           Lib.Password
 import           Lib.ProductKey
 import           Lib.Ssl
 import           Lib.SystemPaths
+import           Lib.Tor
 import           Model
 import           Settings
 
@@ -46,8 +50,11 @@ postRegisterR = handleS9ErrT $ do
 
     -- Check for existing registration.
     checkExistingPasswordRegistration rootAccountName >>= \case
-        Nothing -> pure ()
+        Nothing        -> pure ()
-        Just _  -> sendResponseStatus (Status 209 "Preexisting") ()
+        Just claimedAt -> do
+            res <- mapExceptT (liftIO . runM . injectFilesystemBaseFromContext settings)
+                $ getRegistration productKey claimedAt
+            sendResponseStatus (Status 209 "Preexisting") res
 
     -- install new tor hidden service key and restart tor
     registerResTorAddress <- runM (injectFilesystemBaseFromContext settings $ bootupTor torKeyFileContents) >>= \case
@@ -139,3 +146,21 @@ produceProofOfKey key message = do
     salt <- random16
     let hmac = computeHmac key message salt
     pure $ HmacSig hmac message salt
+
+getRegistration :: (MonadIO m, HasFilesystemBase sig m, Has (Error S9Error) sig m) => Text -> UTCTime -> m RegisterRes
+getRegistration productKey registerResClaimedAt = do
+    torAddress <- getAgentHiddenServiceUrlMaybe >>= \case
+        Nothing -> throwError $ NotFoundE "prior registration" "torAddress"
+        Just t  -> pure $ t
+    caCert <- readSystemPath rootCaCertPath >>= \case
+        Nothing -> throwError $ NotFoundE "prior registration" "cert"
+        Just t  -> pure t
+
+    -- create an hmac of the torAddress + caCert for front end
+    registerResTorAddressSig <- produceProofOfKey productKey torAddress
+    registerResCertSig       <- produceProofOfKey productKey caCert
+
+    let registerResCertName = root_CA_CERT_NAME
+    registerResLanAddress <- getStart9AgentHostnameLocal
+
+    pure RegisterRes { .. }

agent/src/Handler/Status.hs

@@ -28,6 +28,9 @@ import           Lib.SystemPaths         hiding ( (</>) )
 import           Lib.Tor
 import           Settings
 import           Control.Carrier.Lift           ( runM )
+import           System.Process
+import qualified UnliftIO
+import           System.FileLock
 
 getVersionR :: Handler AppVersionRes
 getVersionR = pure . AppVersionRes $ agentVersion
@@ -35,8 +38,7 @@ getVersionR = pure . AppVersionRes $ agentVersion
 getVersionLatestR :: Handler VersionLatestRes
 getVersionLatestR = handleS9ErrT $ do
     s <- getsYesod appSettings
-    v <- interp s $ Reg.getLatestAgentVersion
+    uncurry VersionLatestRes <$> interp s Reg.getLatestAgentVersion
-    pure $ VersionLatestRes v
     where interp s = ExceptT . liftIO . runError . injectFilesystemBaseFromContext s . runRegistryUrlIOC
 
 
@@ -48,6 +50,8 @@ getSpecsR = handleS9ErrT $ do
     specsDisk       <- fmap show . metricDiskSize <$> getDfMetrics
     specsNetworkId  <- lift . runM . injectFilesystemBaseFromContext settings $ getStart9AgentHostname
     specsTorAddress <- lift . runM . injectFilesystemBaseFromContext settings $ getAgentHiddenServiceUrl
+    specsLanAddress <-
+        fmap (<> ".local") . lift . runM . injectFilesystemBaseFromContext settings $ getStart9AgentHostname
 
     let specsAgentVersion = agentVersion
     returnJsonEncoding SpecsRes { .. }
@@ -69,3 +73,9 @@ patchServerR = do
 getGitR :: Handler Text
 getGitR = pure $embedGitRevision
 
+getLogsR :: Handler (JSONResponse [Text])
+getLogsR = do
+    let debugLock = "/root/agent/tmp/debug.lock"
+    UnliftIO.bracket (liftIO $ lockFile debugLock Exclusive) (liftIO . unlockFile) $ const $ do
+        liftIO $ callCommand "journalctl -u agent --since \"1 hour ago\" > /root/agent/tmp/debug.log"
+        liftIO $ JSONResponse . lines <$> readFile "/root/agent/tmp/debug.log"

agent/src/Handler/Types/Apps.hs

@@ -9,6 +9,7 @@ import           Data.Aeson
 import           Data.Aeson.Flatten
 import           Data.Singletons
 
+import qualified Lib.External.AppManifest      as Manifest
 import           Lib.TyFam.ConditionalData
 import           Lib.Types.Core
 import           Lib.Types.Emver
@@ -28,6 +29,7 @@ data AppAvailablePreview = AppAvailablePreview
     , appAvailablePreviewVersionLatest    :: Version
     , appAvailablePreviewDescriptionShort :: Text
     , appAvailablePreviewInstallInfo      :: Maybe (Version, AppStatus)
+    , appAvailablePreviewTimestamp        :: UTCTime
     }
     deriving (Eq, Show)
 instance ToJSON AppAvailablePreview where
@@ -36,6 +38,7 @@ instance ToJSON AppAvailablePreview where
         , "descriptionShort" .= appAvailablePreviewDescriptionShort
         , "versionInstalled" .= (fst <$> appAvailablePreviewInstallInfo)
         , "status" .= (snd <$> appAvailablePreviewInstallInfo)
+        , "latestVersionTimestamp" .= appAvailablePreviewTimestamp
         ]
 
 data AppInstalledPreview = AppInstalledPreview
@@ -43,6 +46,9 @@ data AppInstalledPreview = AppInstalledPreview
     , appInstalledPreviewStatus           :: AppStatus
     , appInstalledPreviewVersionInstalled :: Version
     , appInstalledPreviewTorAddress       :: Maybe TorAddress
+    , appInstalledPreviewLanAddress       :: Maybe LanAddress
+    , appInstalledPreviewTorUi            :: Bool
+    , appInstalledPreviewLanUi            :: Bool
     }
     deriving (Eq, Show)
 instance ToJSON AppInstalledPreview where
@@ -50,6 +56,9 @@ instance ToJSON AppInstalledPreview where
         [ "status" .= appInstalledPreviewStatus
         , "versionInstalled" .= appInstalledPreviewVersionInstalled
         , "torAddress" .= (unTorAddress <$> appInstalledPreviewTorAddress)
+        , "lanAddress" .= (unLanAddress <$> appInstalledPreviewLanAddress)
+        , "torUi" .= appInstalledPreviewTorUi
+        , "lanUi" .= appInstalledPreviewLanUi
         ]
 
 data InstallNewAppReq = InstallNewAppReq
@@ -65,11 +74,14 @@ instance FromJSON InstallNewAppReq where
 
 data AppAvailableFull = AppAvailableFull
     { appAvailableFullBase                   :: AppBase
+    , appAvailableFullLicenseName            :: Maybe Text
+    , appAvailableFullLicenseLink            :: Maybe Text
     , appAvailableFullInstallInfo            :: Maybe (Version, AppStatus)
     , appAvailableFullVersionLatest          :: Version
     , appAvailableFullDescriptionShort       :: Text
     , appAvailableFullDescriptionLong        :: Text
     , appAvailableFullReleaseNotes           :: Text
+    , appAvailableFullInstallAlert           :: Maybe Text
     , appAvailableFullDependencyRequirements :: [Full AppDependencyRequirement]
     , appAvailableFullVersions               :: NonEmpty Version
     }
@@ -78,7 +90,9 @@ instance ToJSON AppAvailableFull where
     toJSON AppAvailableFull {..} = mergeTo
         (toJSON appAvailableFullBase)
         (object
-            [ "versionInstalled" .= fmap fst appAvailableFullInstallInfo
+            [ "licenseName" .= appAvailableFullLicenseName
+            , "licenseLink" .= appAvailableFullLicenseLink
+            , "versionInstalled" .= fmap fst appAvailableFullInstallInfo
             , "status" .= fmap snd appAvailableFullInstallInfo
             , "versionLatest" .= appAvailableFullVersionLatest
             , "descriptionShort" .= appAvailableFullDescriptionShort
@@ -86,6 +100,7 @@ instance ToJSON AppAvailableFull where
             , "versions" .= appAvailableFullVersions
             , "releaseNotes" .= appAvailableFullReleaseNotes
             , "serviceRequirements" .= appAvailableFullDependencyRequirements
+            , "installAlert" .= appAvailableFullInstallAlert
             ]
         )
 
@@ -120,12 +135,21 @@ instance ToJSON (AppDependencyRequirement Keep) where
 -- mute violations downstream of version for installing apps
 data AppInstalledFull = AppInstalledFull
     { appInstalledFullBase                   :: AppBase
+    , appInstalledFullLicenseName            :: Maybe Text
+    , appInstalledFullLicenseLink            :: Maybe Text
     , appInstalledFullStatus                 :: AppStatus
     , appInstalledFullVersionInstalled       :: Version
     , appInstalledFullTorAddress             :: Maybe TorAddress
+    , appInstalledFullLanAddress             :: Maybe LanAddress
+    , appInstalledFullTorUi                  :: Bool
+    , appInstalledFullLanUi                  :: Bool
     , appInstalledFullInstructions           :: Maybe Text
     , appInstalledFullLastBackup             :: Maybe UTCTime
     , appInstalledFullConfiguredRequirements :: [Stripped AppDependencyRequirement]
+    , appInstalledFullUninstallAlert         :: Maybe Text
+    , appInstalledFullRestoreAlert           :: Maybe Text
+    , appInstalledFullStartAlert             :: Maybe Text
+    , appInstalledFullActions                :: [Manifest.Action]
     }
 instance ToJSON AppInstalledFull where
     toJSON AppInstalledFull {..} = object
@@ -133,23 +157,34 @@ instance ToJSON AppInstalledFull where
         , "lastBackup" .= appInstalledFullLastBackup
         , "configuredRequirements" .= appInstalledFullConfiguredRequirements
         , "torAddress" .= (unTorAddress <$> appInstalledFullTorAddress)
+        , "lanAddress" .= (unLanAddress <$> appInstalledFullLanAddress)
+        , "torUi" .= appInstalledFullTorUi
+        , "lanUi" .= appInstalledFullLanUi
         , "id" .= appBaseId appInstalledFullBase
         , "title" .= appBaseTitle appInstalledFullBase
+        , "licenseName" .= appInstalledFullLicenseName
+        , "licenseLink" .= appInstalledFullLicenseLink
         , "iconURL" .= appBaseIconUrl appInstalledFullBase
         , "versionInstalled" .= appInstalledFullVersionInstalled
         , "status" .= appInstalledFullStatus
+        , "uninstallAlert" .= appInstalledFullUninstallAlert
+        , "restoreAlert" .= appInstalledFullRestoreAlert
+        , "startAlert" .= appInstalledFullStartAlert
+        , "actions" .= appInstalledFullActions
         ]
 
 data AppVersionInfo = AppVersionInfo
     { appVersionInfoVersion                :: Version
     , appVersionInfoReleaseNotes           :: Text
     , appVersionInfoDependencyRequirements :: [Full AppDependencyRequirement]
+    , appVersionInfoInstallAlert           :: Maybe Text
     }
 instance ToJSON AppVersionInfo where
     toJSON AppVersionInfo {..} = object
         [ "version" .= appVersionInfoVersion
         , "releaseNotes" .= appVersionInfoReleaseNotes
         , "serviceRequirements" .= appVersionInfoDependencyRequirements
+        , "installAlert" .= appVersionInfoInstallAlert
         ]
 
 data ApiDependencyViolation

agent/src/Handler/Types/V0/Base.hs

@@ -15,11 +15,13 @@ import           Lib.Types.Emver
 import           Model
 
 data VersionLatestRes = VersionLatestRes
-    { versionLatestVersion :: Version
+    { versionLatestVersion      :: Version
+    , versionLatestReleaseNotes :: Maybe Text
     }
     deriving (Eq, Show)
 instance ToJSON VersionLatestRes where
-    toJSON VersionLatestRes {..} = object $ ["versionLatest" .= versionLatestVersion]
+    toJSON VersionLatestRes {..} =
+        object $ ["versionLatest" .= versionLatestVersion, "releaseNotes" .= versionLatestReleaseNotes]
 instance ToTypedContent VersionLatestRes where
     toTypedContent = toTypedContent . toJSON
 instance ToContent VersionLatestRes where
@@ -36,6 +38,8 @@ data ServerRes = ServerRes
     , serverSsh                    :: [SshKeyFingerprint]
     , serverAlternativeRegistryUrl :: Maybe Text
     , serverSpecs                  :: SpecsRes
+    , serverWelcomeAck             :: Bool
+    , serverAutoCheckUpdates       :: Bool
     }
     deriving (Eq, Show)
 
@@ -51,12 +55,13 @@ instance ToJSON ServerRes where
             Nothing   -> String "UPDATING"
             Just stat -> toJSON stat
         , "versionInstalled" .= serverVersionInstalled
-        , "versionLatest" .= Null
         , "notifications" .= serverNotifications
         , "wifi" .= serverWifi
         , "ssh" .= serverSsh
         , "alternativeRegistryUrl" .= serverAlternativeRegistryUrl
         , "specs" .= serverSpecs
+        , "welcomeAck" .= serverWelcomeAck
+        , "autoCheckUpdates" .= serverAutoCheckUpdates
         ]
 instance ToTypedContent ServerRes where
     toTypedContent = toTypedContent . toJSON

agent/src/Handler/Types/V0/Specs.hs

@@ -16,6 +16,7 @@ data SpecsRes = SpecsRes
     , specsNetworkId    :: Text
     , specsAgentVersion :: Version
     , specsTorAddress   :: Text
+    , specsLanAddress   :: Text
     }
     deriving (Eq, Show)
 
@@ -23,6 +24,7 @@ instance ToJSON SpecsRes where
     toJSON SpecsRes {..} = object
         [ "EmbassyOS Version" .= specsAgentVersion
         , "Tor Address" .= specsTorAddress
+        , "LAN Address" .= specsLanAddress
         , "Network ID" .= specsNetworkId
         , "CPU" .= specsCPU
         , "Memory" .= specsMem
@@ -33,6 +35,7 @@ instance ToJSON SpecsRes where
             . fold
             $ [ "EmbassyOS Version" .= specsAgentVersion
               , "Tor Address" .= specsTorAddress
+              , "LAN Address" .= specsLanAddress
               , "Network ID" .= specsNetworkId
               , "CPU" .= specsCPU
               , "Memory" .= specsMem

agent/src/Handler/V0.hs

@@ -8,7 +8,7 @@ import           Control.Carrier.Lift           ( runM )
 import           Data.Aeson
 import           Data.IORef
 import qualified Data.Text                     as T
-import           Database.Persist
+import           Database.Persist              as Persist
 import           Yesod.Core.Handler
 import           Yesod.Persist.Core
 import           Yesod.Core.Json
@@ -30,6 +30,7 @@ import           Lib.SystemPaths
 import           Lib.Ssh
 import           Lib.Tor
 import           Lib.Types.Core
+import           Lib.Types.Emver
 import           Model
 import           Settings
 import           Util.Function
@@ -37,7 +38,8 @@ import           Util.Function
 
 getServerR :: Handler (JsonEncoding ServerRes)
 getServerR = handleS9ErrT $ do
-    settings   <- getsYesod appSettings
+    agentCtx <- getYesod
+    let settings = appSettings agentCtx
     now        <- liftIO getCurrentTime
     isUpdating <- getsYesod appIsUpdating >>= liftIO . readIORef
 
@@ -53,8 +55,11 @@ getServerR = handleS9ErrT $ do
     alternativeRegistryUrl <- runM $ injectFilesystemBaseFromContext settings $ readSystemPath altRegistryUrlPath
     name                   <- runM $ injectFilesystemBaseFromContext settings $ readSystemPath serverNamePath
     ssh                    <- readFromPath settings sshKeysFilePath >>= parseSshKeys
-    wifi                   <- WpaSupplicant.runWlan0 $ liftA2 WifiList WpaSupplicant.getCurrentNetwork WpaSupplicant.listNetworks
+    wifi <- WpaSupplicant.runWlan0 $ liftA2 WifiList WpaSupplicant.getCurrentNetwork WpaSupplicant.listNetworks
     specs                  <- getSpecs settings
+    welcomeAck             <- fmap isJust . lift . runDB . Persist.get $ WelcomeAckKey agentVersion
+    autoCheckUpdates       <- runM $ injectFilesystemBaseFromContext settings $ fmap not (existsSystemPath disableAutoCheckUpdatesPath)
+
     let sid = T.drop 7 $ specsNetworkId specs
 
     jsonEncode ServerRes { serverId                     = specsNetworkId specs
@@ -67,6 +72,8 @@ getServerR = handleS9ErrT $ do
                          , serverSsh                    = ssh
                          , serverAlternativeRegistryUrl = alternativeRegistryUrl
                          , serverSpecs                  = specs
+                         , serverWelcomeAck             = welcomeAck
+                         , serverAutoCheckUpdates       = autoCheckUpdates
                          }
     where
         parseSshKeys :: Text -> S9ErrT Handler [SshKeyFingerprint]
@@ -76,6 +83,9 @@ getServerR = handleS9ErrT $ do
                 Left  e  -> throwE $ InvalidSshKeyE (toS e)
                 Right as -> pure $ uncurry3 SshKeyFingerprint <$> as
 
+postWelcomeR :: Version -> Handler ()
+postWelcomeR version = runDB $ repsert (WelcomeAckKey version) WelcomeAck
+
 getSpecs :: MonadIO m => AppSettings -> S9ErrT m SpecsRes
 getSpecs settings = do
     specsCPU        <- liftIO getCpuInfo
@@ -83,6 +93,7 @@ getSpecs settings = do
     specsDisk       <- fmap show . metricDiskSize <$> getDfMetrics
     specsNetworkId  <- runM $ injectFilesystemBaseFromContext settings getStart9AgentHostname
     specsTorAddress <- runM $ injectFilesystemBaseFromContext settings getAgentHiddenServiceUrl
+    specsLanAddress <- fmap (<> ".local") . runM $ injectFilesystemBaseFromContext settings getStart9AgentHostname
 
     let specsAgentVersion = agentVersion
     pure $ SpecsRes { .. }
@@ -102,9 +113,22 @@ newtype NullablePatchReq = NullablePatchReq { mpatchValue :: Maybe Text } derivi
 instance FromJSON NullablePatchReq where
     parseJSON = withObject "Nullable Patch Request" $ \o -> NullablePatchReq <$> o .:? "value"
 
+newtype BoolPatchReq = BoolPatchReq { bpatchValue :: Bool } deriving (Eq, Show)
+
+instance FromJSON BoolPatchReq where
+    parseJSON = withObject "Patch Request" $ \o -> BoolPatchReq <$> o .: "value"
+
 patchNameR :: Handler ()
 patchNameR = patchFile serverNamePath
 
+patchAutoCheckUpdatesR :: Handler ()
+patchAutoCheckUpdatesR = do
+    settings         <- getsYesod appSettings
+    BoolPatchReq val <- requireCheckJsonBody
+    runM $ injectFilesystemBaseFromContext settings $ if val
+        then deleteSystemPath disableAutoCheckUpdatesPath
+        else writeSystemPath disableAutoCheckUpdatesPath ""
+
 patchFile :: SystemPath -> Handler ()
 patchFile path = do
     settings     <- getsYesod appSettings

agent/src/Lib/Algebra/Domain/AppMgr.hs

@@ -11,8 +11,7 @@ module Lib.Algebra.Domain.AppMgr
     ( module Lib.Algebra.Domain.AppMgr
     , module Lib.Algebra.Domain.AppMgr.Types
     , module Lib.Algebra.Domain.AppMgr.TH
-    )
+    ) where
-where
 
 import           Startlude
 
@@ -25,32 +24,32 @@ import qualified Data.HashMap.Strict           as HM
 import           Data.Singletons.Prelude hiding ( Error )
 import           Data.Singletons.Prelude.Either
 import qualified Data.String                   as String
-import           Exinst
 
-import           Lib.Algebra.Domain.AppMgr.Types
-import           Lib.Algebra.Domain.AppMgr.TH
-import           Lib.Error
-import           Lib.External.AppManifest
-import           Lib.TyFam.ConditionalData
-import           Lib.Types.Core                 ( AppId(..)
-                                                , AppContainerStatus(..)
-                                                )
-import           Lib.Types.NetAddress
-import           Lib.Types.Emver
-import           Control.Monad.Trans.Class      ( MonadTrans )
-import qualified Data.ByteString.Lazy          as LBS
-import           System.Process.Typed
-import           Data.String.Interpolate.IsString
-                                                ( i )
 import           Control.Monad.Base             ( MonadBase(..) )
 import           Control.Monad.Fail             ( MonadFail(fail) )
-import           Control.Monad.Trans.Resource   ( MonadResource(..) )
+import           Control.Monad.Trans.Class      ( MonadTrans )
-import           Control.Monad.Trans.Control    ( defaultLiftBaseWith
+import           Control.Monad.Trans.Control    ( MonadBaseControl(..)
-                                                , defaultRestoreM
                                                 , MonadTransControl(..)
-                                                , MonadBaseControl(..)
+                                                , defaultLiftBaseWith
+                                                , defaultRestoreM
                                                 )
+import           Control.Monad.Trans.Resource   ( MonadResource(..) )
 import qualified Data.ByteString.Char8         as C8
+import qualified Data.ByteString.Lazy          as LBS
+import           Data.String.Interpolate.IsString
+                                                ( i )
+import           Lib.Algebra.Domain.AppMgr.TH
+import           Lib.Algebra.Domain.AppMgr.Types
+import           Lib.Error
+import qualified Lib.External.AppManifest      as Manifest
+import           Lib.TyFam.ConditionalData
+import           Lib.Types.Core                 ( AppContainerStatus(..)
+                                                , AppId(..)
+                                                )
+import           Lib.Types.Emver
+import           Lib.Types.NetAddress
+import           System.Process
+import           System.Process.Typed
 
 
 type InfoRes :: Either OnlyInfoFlag [IncludeInfoFlag] -> Type
@@ -67,7 +66,7 @@ data InfoRes a = InfoRes
               (Either_ (DefaultEqSym1 'OnlyDependencies) (ElemSym1 'IncludeDependencies) a)
               (HM.HashMap AppId DependencyInfo)
     , infoResManifest
-          :: Include (Either_ (DefaultEqSym1 'OnlyManifest) (ElemSym1 'IncludeManifest) a) (Some1 AppManifest)
+          :: Include (Either_ (DefaultEqSym1 'OnlyManifest) (ElemSym1 'IncludeManifest) a) Manifest.AppManifest
     , infoResStatus :: Include (Either_ (DefaultEqSym1 'OnlyStatus) (ElemSym1 'IncludeStatus) a) AppContainerStatus
     }
 instance SingI (a :: Either OnlyInfoFlag [IncludeInfoFlag]) => FromJSON (InfoRes a) where
@@ -271,6 +270,8 @@ data AppMgr (m :: Type -> Type) k where
     -- Tor ::_
     Update ::DryRun -> AppId -> Maybe VersionRange -> AppMgr m BreakageMap
     -- Verify ::_
+    LanEnable ::AppMgr m ()
+    Action ::AppId -> Text -> AppMgr m (HM.HashMap Text Value)
 makeSmartConstructors ''AppMgr
 
 newtype AppMgrCliC m a = AppMgrCliC { runAppMgrCliC :: m a }
@@ -369,13 +370,16 @@ instance (Has (Error S9Error) sig m, Algebra sig m, MonadIO m) => Algebra (AppMg
         (L (List (SRight flags))) -> do
             let renderedFlags = (genInclusiveFlag <$> fromSing flags) <> ["--json"]
             let args          = "list" : renderedFlags
-            (ec, out) <- readProcessInheritStderr "appmgr" args ""
+            let runIt retryCount = do
-            res       <- case ec of
+                    (ec, out) <- readProcessInheritStderr "appmgr" args ""
-                ExitSuccess -> case withSingI flags $ eitherDecodeStrict out of
+                    case ec of
-                    Left  e -> throwError $ AppMgrParseE (toS $ String.unwords args) (decodeUtf8 out) e
+                        ExitSuccess -> case withSingI flags $ eitherDecodeStrict out of
-                    Right x -> pure x
+                            Left  e -> throwError $ AppMgrParseE (toS $ String.unwords args) (decodeUtf8 out) e
-                ExitFailure n -> throwError $ AppMgrE "list" n
+                            Right x -> pure $ ctx $> x
-            pure $ ctx $> res
+                        ExitFailure 6 ->
+                            if retryCount > 0 then runIt (retryCount - 1) else throwError $ AppMgrE "list" 6
+                        ExitFailure n -> throwError $ AppMgrE "list" n
+            runIt (1 :: Word) -- with 1 retry
         (L (Remove dryorpurge appId)) -> do
             let args = "remove" : case dryorpurge of
                     Left  (DryRun True) -> ["--dry-run", show appId, "--json"]
@@ -422,6 +426,16 @@ instance (Has (Error S9Error) sig m, Algebra sig m, MonadIO m) => Algebra (AppMg
                 ExitFailure 6 ->
                     throwError $ NotFoundE "appId@version" ([i|#{appId}#{maybe "" (('@':) . show) version}|])
                 ExitFailure n -> throwError $ AppMgrE (toS $ String.unwords args) n
+        (L LanEnable            ) -> liftIO $ callProcess "appmgr" ["lan", "enable"] $> ctx
+        (L (Action appId action)) -> do
+            let args = ["actions", show appId, toS action]
+            (ec, out) <- readProcessInheritStderr "appmgr" args ""
+            case ec of
+                ExitSuccess -> case eitherDecodeStrict out of
+                    Left  e -> throwError $ AppMgrParseE (toS $ String.unwords args) (decodeUtf8 out) e
+                    Right x -> pure $ ctx $> x
+                ExitFailure 6 -> throwError $ NotFoundE "appId" (show appId)
+                ExitFailure n -> throwError $ AppMgrE (toS $ String.unwords args) n
         R other -> AppMgrCliC $ alg (runAppMgrCliC . hdl) other ctx
         where
             versionSpec :: (IsString a, Semigroup a, ConvertText String a) => Maybe VersionRange -> a -> a

agent/src/Lib/Error.hs

@@ -31,6 +31,7 @@ data S9Error =
     | AppMgrParseE Text Text String
     | AppMgrInvalidConfigE Text
     | AppMgrE Text Int
+    | EjectE Int
     | AvahiE Text
     | MetricE Text
     | AppMgrVersionE Version VersionRange
@@ -51,6 +52,7 @@ data S9Error =
     | WifiOrphaningE
     | NoPasswordExistsE
     | HostsParamsE Text
+    | ParamsE Text
     | MissingFileE SystemPath
     | ClientCryptographyE Text
     | TTLExpirationE Text
@@ -86,6 +88,7 @@ toError = \case
     AppMgrParseE cmd result e ->
         ErrorResponse APPMGR_PARSE_ERROR [i|"appmgr #{cmd}" yielded an unparseable result:#{result}\nError: #{e}|]
     AppMgrE cmd code -> ErrorResponse APPMGR_ERROR [i|"appmgr #{cmd}" exited with #{code}|]
+    EjectE code -> ErrorResponse EJECT_ERROR [i|"eject" command exited with #{code}|]
     AppMgrVersionE av avs ->
         ErrorResponse APPMGR_ERROR [i|"appmgr version #{av}" fails to satisfy requisite spec #{avs}|]
     AvahiE  e               -> ErrorResponse AVAHI_ERROR [i|#{e}|]
@@ -136,6 +139,7 @@ toError = \case
     TTLExpirationE      desc  -> ErrorResponse REGISTRATION_ERROR [i|TTL Expiration failure: #{desc}|]
     EnvironmentValE     appId -> ErrorResponse SYNCHRONIZATION_ERROR [i|Could not read environment values for #{appId}|]
     HostsParamsE        key   -> ErrorResponse REGISTRATION_ERROR [i|Missing or invalid parameter #{key}|]
+    ParamsE             key   -> ErrorResponse INVALID_REQUEST [i|Missing or invalid parameter #{key}|]
     InternalE           msg   -> ErrorResponse INTERNAL_ERROR msg
     BackupE appId reason      -> ErrorResponse BACKUP_ERROR [i|Backup failed for #{appId}: #{reason}|]
     BackupPassInvalidE        -> ErrorResponse BACKUP_ERROR [i|Password provided for backups is invalid|]
@@ -151,6 +155,7 @@ data ErrorCode =
     | APPMGR_CONFIG_ERROR
     | APPMGR_PARSE_ERROR
     | APPMGR_ERROR
+    | EJECT_ERROR
     | AVAHI_ERROR
     | REGISTRY_ERROR
     | APP_NOT_INSTALLED
@@ -201,6 +206,7 @@ toStatus = \case
     AppMgrParseE{}         -> status500
     AppMgrInvalidConfigE _ -> status400
     AppMgrE        _ _     -> status500
+    EjectE  _              -> status500
     AppMgrVersionE _ _     -> status500
     AvahiE  _              -> status500
     MetricE _              -> status500
@@ -238,6 +244,7 @@ toStatus = \case
     TTLExpirationE      _   -> status403
     EnvironmentValE     _   -> status500
     HostsParamsE        _   -> status400
+    ParamsE             _   -> status400
     BackupE _ _             -> status500
     BackupPassInvalidE      -> status403
     InternalE _             -> status500

agent/src/Lib/External/AppManifest.hs

@@ -6,17 +6,15 @@ import           Startlude               hiding ( ask )
 
 import           Control.Effect.Reader.Labelled
 import           Data.Aeson
-import           Data.Singletons.TypeLits
 import qualified Data.HashMap.Strict           as HM
 import qualified Data.Yaml                     as Yaml
-import           Exinst
 
+import           Control.Monad.Fail             ( MonadFail(fail) )
 import           Lib.Error
 import           Lib.SystemPaths
 import           Lib.Types.Core
 import           Lib.Types.Emver
 import           Lib.Types.Emver.Orphans        ( )
-import           Control.Monad.Fail             ( MonadFail(fail) )
 
 data ImageType = ImageTypeTar
     deriving (Eq, Show)
@@ -49,52 +47,111 @@ instance FromJSON AssetMapping where
         assetMappingOverwrite <- o .: "overwrite"
         pure $ AssetMapping { .. }
 
-data AppManifest (n :: Nat) where
+data Action = Action
-    AppManifestV0 ::{ appManifestV0Id :: AppId
+    { actionId              :: Text
-                    , appManifestV0Version :: Version
+    , actionName            :: Text
-                    , appManifestV0Title :: Text
+    , actionDescription     :: Text
-                    , appManifestV0DescShort :: Text
+    , actionWarning         :: Maybe Text
-                    , appManifestV0DescLong :: Text
+    , actionAllowedStatuses :: [AppContainerStatus]
-                    , appManifestV0ReleaseNotes :: Text
+    }
-                    , appManifestV0PortMapping :: HM.HashMap Word16 Word16
+    deriving Show
-                    , appManifestV0ImageType :: ImageType
+instance FromJSON Action where
-                    , appManifestV0Mount :: FilePath
+    parseJSON = withObject "AppAction" $ \o -> do
-                    , appManifestV0Assets :: [AssetMapping]
+        actionId              <- o .: "id"
-                    , appManifestV0OnionVersion :: OnionVersion
+        actionName            <- o .: "name"
-                    , appManifestV0Dependencies :: HM.HashMap AppId VersionRange
+        actionDescription     <- o .: "description"
-                    } -> AppManifest 0
+        actionWarning         <- o .:? "warning"
+        actionAllowedStatuses <- o .: "allowed-statuses"
+        pure Action { .. }
+instance ToJSON Action where
+    toJSON Action {..} =
+        object
+            $  [ "id" .= actionId
+               , "name" .= actionName
+               , "description" .= actionDescription
+               , "allowedStatuses" .= actionAllowedStatuses
+               ]
+            <> maybeToList (("warning" .=) <$> actionWarning)
 
-instance FromJSON (Some1 AppManifest) where
-    parseJSON = withObject "App Manifest" $ \o -> do
-        o .: "compat" >>= \case
-            ("v0" :: Text) -> Some1 (SNat @0) <$> parseJSON (Object o)
-            compat         -> fail $ "Unknown Manifest Version: " <> toS compat
 
-instance FromJSON (AppManifest 0) where
+data AppManifest where
-    parseJSON = withObject "App Manifest V0" $ \o -> do
+     AppManifest ::{ appManifestId :: AppId
-        appManifestV0Id           <- o .: "id"
+                    , appManifestVersion :: Version
-        appManifestV0Version      <- o .: "version"
+                    , appManifestTitle :: Text
-        appManifestV0Title        <- o .: "title"
+                    , appManifestLicenseName :: Maybe Text
-        appManifestV0DescShort    <- o .: "description" >>= (.: "short")
+                    , appManifestLicenseLink :: Maybe Text
-        appManifestV0DescLong     <- o .: "description" >>= (.: "long")
+                    , appManifestDescShort :: Text
-        appManifestV0ReleaseNotes <- o .: "release-notes"
+                    , appManifestDescLong :: Text
-        appManifestV0PortMapping  <- o .: "ports" >>= fmap HM.fromList . traverse parsePortMapping
+                    , appManifestReleaseNotes :: Text
-        appManifestV0ImageType    <- o .: "image" >>= (.: "type")
+                    , appManifestPortMapping :: [PortMapEntry]
-        appManifestV0Mount        <- o .: "mount"
+                    , appManifestImageType :: ImageType
-        appManifestV0Assets       <- o .: "assets" >>= traverse parseJSON
+                    , appManifestMount :: FilePath
-        appManifestV0OnionVersion <- o .: "hidden-service-version"
+                    , appManifestAssets :: [AssetMapping]
-        appManifestV0Dependencies <- o .:? "dependencies" .!= HM.empty >>= traverse parseDepInfo
+                    , appManifestOnionVersion :: OnionVersion
-        pure $ AppManifestV0 { .. }
+                    , appManifestDependencies :: HM.HashMap AppId VersionRange
-        where
+                    , appManifestUninstallAlert :: Maybe Text
-            parsePortMapping = withObject "Port Mapping" $ \o -> liftA2 (,) (o .: "tor") (o .: "internal")
+                    , appManifestRestoreAlert   :: Maybe Text
-            parseDepInfo     = withObject "Dep Info" $ (.: "version")
+                    , appManifestStartAlert :: Maybe Text
+                    , appManifestActions :: [Action]
+                    } -> AppManifest
+deriving instance Show AppManifest
 
-getAppManifest :: (MonadIO m, HasFilesystemBase sig m) => AppId -> S9ErrT m (Maybe (Some1 AppManifest))
+torUiAvailable :: AppManifest -> Bool
+torUiAvailable AppManifest {..} = any (== 80) $ portMapEntryTor <$> appManifestPortMapping
+
+lanUiAvailable :: AppManifest -> Bool
+lanUiAvailable AppManifest {..} = any id $ fmap portMapEntryLan appManifestPortMapping <&> \case
+    Just Standard     -> True
+    Just (Custom 443) -> True
+    Just (Custom 80 ) -> True
+    _                 -> False
+
+instance FromJSON AppManifest where
+    parseJSON = withObject "App Manifest " $ \o -> do
+        appManifestId             <- o .: "id"
+        appManifestVersion        <- o .: "version"
+        appManifestTitle          <- o .: "title"
+        appManifestLicenseName    <- o .:? "license-info" >>= traverse (.: "license")
+        appManifestLicenseLink    <- o .:? "license-info" >>= traverse (.: "url")
+        appManifestDescShort      <- o .: "description" >>= (.: "short")
+        appManifestDescLong       <- o .: "description" >>= (.: "long")
+        appManifestReleaseNotes   <- o .: "release-notes"
+        appManifestPortMapping    <- o .: "ports"
+        appManifestImageType      <- o .: "image" >>= (.: "type")
+        appManifestMount          <- o .: "mount"
+        appManifestAssets         <- o .: "assets" >>= traverse parseJSON
+        appManifestOnionVersion   <- o .: "hidden-service-version"
+        appManifestDependencies   <- o .:? "dependencies" .!= HM.empty >>= traverse parseDepInfo
+        appManifestUninstallAlert <- o .:? "uninstall-alert"
+        appManifestRestoreAlert   <- o .:? "restore-alert"
+        appManifestStartAlert     <- o .:? "start-alert"
+        appManifestActions        <- o .: "actions"
+        pure $ AppManifest { .. }
+        where parseDepInfo = withObject "Dep Info" $ (.: "version")
+
+getAppManifest :: (MonadIO m, HasFilesystemBase sig m) => AppId -> S9ErrT m (Maybe AppManifest)
 getAppManifest appId = do
     base <- ask @"filesystemBase"
     ExceptT $ first (ManifestParseE appId) <$> liftIO
         (Yaml.decodeFileEither . toS $ (appMgrAppPath appId <> "manifest.yaml") `relativeTo` base)
 
-uiAvailable :: AppManifest n -> Bool
+data LanConfiguration = Standard | Custom Word16 deriving (Eq, Show)
-uiAvailable = \case
+instance FromJSON LanConfiguration where
-    AppManifestV0 { appManifestV0PortMapping } -> elem 80 (HM.keys appManifestV0PortMapping)
+    parseJSON = liftA2 (<|>) standard custom
+        where
+            standard =
+                withText "Standard Lan" \t -> if t == "standard" then pure Standard else fail "Not Standard Lan Conf"
+            custom = withObject "Custom Lan" $ \o -> do
+                Custom <$> (o .: "custom" >>= (.: "port"))
+data PortMapEntry = PortMapEntry
+    { portMapEntryInternal :: Word16
+    , portMapEntryTor      :: Word16
+    , portMapEntryLan      :: Maybe LanConfiguration
+    }
+    deriving (Eq, Show)
+instance FromJSON PortMapEntry where
+    parseJSON = withObject "Port Map Entry" $ \o -> do
+        portMapEntryInternal <- o .: "internal"
+        portMapEntryTor      <- o .: "tor"
+        portMapEntryLan      <- o .:? "lan"
+        pure PortMapEntry { .. }

agent/src/Lib/External/Registry.hs

@@ -13,8 +13,8 @@ import           Startlude.ByteStream    hiding ( count )
 
 import           Conduit
 import           Control.Algebra
-import           Control.Effect.Lift
 import           Control.Effect.Error
+import           Control.Effect.Lift
 import           Control.Effect.Reader.Labelled
 import           Control.Monad.Fail             ( fail )
 import           Control.Monad.Trans.Resource
@@ -30,14 +30,17 @@ import           System.Directory
 import           System.Process
 
 import           Constants
+import qualified Data.Aeson.Types               ( parseEither )
+import           Data.Time.ISO8601              ( parseISO8601 )
 import           Lib.Algebra.State.RegistryUrl
 import           Lib.Error
+import           Lib.External.AppManifest
 import           Lib.SystemPaths
 import           Lib.Types.Core
 import           Lib.Types.Emver
 import           Lib.Types.ServerApp
 
-newtype AppManifestRes = AppManifestRes
+newtype AppIndexRes = AppIndexRes
     { storeApps :: [StoreApp] } deriving (Eq, Show)
 
 newtype RegistryVersionForSpecRes = RegistryVersionForSpecRes
@@ -84,8 +87,8 @@ getLifelineBinary avs = do
     liftIO $ runConduitRes $ httpSource request getResponseBody .| sinkFile (toS lifelineTarget)
     liftIO $ void $ readProcessWithExitCode "chmod" ["700", toS lifelineTarget] ""
 
-getAppManifest :: (MonadIO m, Has (Error S9Error) sig m, Has RegistryUrl sig m) => m AppManifestRes
+getAppIndex :: (MonadIO m, Has (Error S9Error) sig m, Has RegistryUrl sig m) => m AppIndexRes
-getAppManifest = do
+getAppIndex = do
     manifestPath <- registryManifestUrl
     req          <- liftIO $ fmap setUserAgent . parseRequestThrow $ toS manifestPath
     val          <- (liftIO . try @SomeException) (httpBS req) >>= \case
@@ -95,22 +98,29 @@ getAppManifest = do
         Left  e -> throwError $ RegistryParseE manifestPath . toS $ e
         Right a -> pure a
 
+getAppManifest :: (MonadIO m, Has (Error S9Error) sig m, Has RegistryUrl sig m) => AppId -> m AppManifest
+getAppManifest appId = do
+    let path = "/apps/manifest/" <> unAppId appId
+    v <- registryRequest path
+    case Data.Aeson.Types.parseEither parseJSON v of
+        Left  e -> throwError $ RegistryParseE path . toS $ e
+        Right a -> pure a
 
 getStoreAppInfo :: (MonadIO m, Has RegistryUrl sig m, Has (Error S9Error) sig m) => AppId -> m (Maybe StoreApp)
-getStoreAppInfo name = find ((== name) . storeAppId) . storeApps <$> getAppManifest
+getStoreAppInfo name = find ((== name) . storeAppId) . storeApps <$> getAppIndex
 
-parseBsManifest :: Has RegistryUrl sig m => ByteString -> m (Either String AppManifestRes)
+parseBsManifest :: Has RegistryUrl sig m => ByteString -> m (Either String AppIndexRes)
 parseBsManifest bs = do
     parseRegistryRes' <- parseRegistryRes
     pure $ parseEither parseRegistryRes' . fromJust . decodeThrow $ bs
 
-parseRegistryRes :: Has RegistryUrl sig m => m (Value -> Parser AppManifestRes)
+parseRegistryRes :: Has RegistryUrl sig m => m (Value -> Parser AppIndexRes)
 parseRegistryRes = do
     parseAppData' <- parseAppData
     pure $ withObject "app registry response" $ \obj -> do
         let keyVals       = HM.toList obj
         let mManifestApps = fmap (\(k, v) -> parseMaybe (parseAppData' (AppId k)) v) keyVals
-        pure . AppManifestRes . catMaybes $ mManifestApps
+        pure . AppIndexRes . catMaybes $ mManifestApps
 
 registryUrl :: (Has RegistryUrl sig m) => m Text
 registryUrl = maybe "https://registry.start9labs.com:443" show <$> getRegistryUrl
@@ -135,6 +145,7 @@ parseAppData = do
         storeAppVersions         <- ad .: "version-info" >>= \case
             []       -> fail "No Valid Version Info"
             (x : xs) -> pure $ x :| xs
+        storeAppTimestamp <- ad .: "timestamp" >>= maybe (fail "Invalid ISO8601 Timestamp") pure . parseISO8601
         pure StoreApp { .. }
 
 getAppVersionForSpec :: (Has RegistryUrl sig m, Has (Error S9Error) sig m, MonadIO m)
@@ -148,12 +159,13 @@ getAppVersionForSpec appId spec = do
         v <- o .: "version"
         pure v
 
-getLatestAgentVersion :: (Has RegistryUrl sig m, Has (Error S9Error) sig m, MonadIO m) => m Version
+getLatestAgentVersion :: (Has RegistryUrl sig m, Has (Error S9Error) sig m, MonadIO m) => m (Version, Maybe Text)
 getLatestAgentVersion = do
     val <- registryRequest agentVersionPath
     parseOrThrow agentVersionPath val $ withObject "version response" $ \o -> do
-        v <- o .: "version"
+        v  <- o .: "version"
-        pure v
+        rn <- o .:? "release-notes"
+        pure (v, rn)
     where agentVersionPath = "sys/version/agent"
 
 getLatestAgentVersionForSpec :: (Has RegistryUrl sig m, Has (Lift IO) sig m, Has (Error S9Error) sig m)

agent/src/Lib/SelfUpdate.hs

@@ -5,7 +5,9 @@
 {-# LANGUAGE TupleSections       #-}
 module Lib.SelfUpdate where
 
-import           Startlude               hiding ( runReader )
+import           Startlude               hiding ( handle
+                                                , runReader
+                                                )
 
 import           Control.Carrier.Error.Either
 import           Control.Lens
@@ -29,6 +31,7 @@ import           Lib.SystemPaths
 import           Lib.Types.Emver
 import           Lib.WebServer
 import           Settings
+import           UnliftIO.Exception             ( handle )
 
 youngAgentPort :: Word16
 youngAgentPort = 5960
@@ -191,18 +194,21 @@ runSyncOps syncOps = do
     pure res
 
 synchronizeSystemState :: AgentCtx -> Version -> IO ()
-synchronizeSystemState ctx _version = handle @SomeException cleanup $ flip runReaderT ctx $ do
+synchronizeSystemState ctx _version = handle @_ @SomeException cleanup $ flip runReaderT ctx $ do
     (restartsAndRuns, mTid) <- case synchronizer of
         Synchronizer { synchronizerOperations } -> flip runStateT Nothing $ for synchronizerOperations $ \syncOp -> do
             shouldRun <- lift $ syncOpShouldRun syncOp
             putStrLn @Text [i|Sync Op "#{syncOpName syncOp}" should run: #{shouldRun}|]
             when shouldRun $ do
-                whenM (isNothing <$> get) $ do
+                tid <- get >>= \case
-                    tid <- liftIO . forkIO . forever $ playSong 300 updateInProgress *> threadDelay 20_000_000
+                    Nothing -> do
-                    put (Just tid)
+                        tid <- liftIO . forkIO . forever $ playSong 300 updateInProgress *> threadDelay 20_000_000
+                        put (Just tid)
+                        pure tid
+                    Just tid -> pure tid
                 putStrLn @Text [i|Running Sync Op: #{syncOpName syncOp}|]
                 setUpdate True
-                lift $ syncOpRun syncOp
+                lift $ handle @_ @SomeException (\e -> lift $ killThread tid *> cleanup e) $ syncOpRun syncOp
             pure $ (syncOpRequiresReboot syncOp, shouldRun)
     case mTid of
         Nothing  -> pure ()
@@ -222,5 +228,6 @@ synchronizeSystemState ctx _version = handle @SomeException cleanup $ flip runRe
             void $ try @SomeException Sound.stop
             void $ try @SomeException Sound.unexport
             let e' = InternalE $ show e
+            setUpdate False
             flip runReaderT ctx $ cantFail $ failUpdate e'
 

agent/src/Lib/Synchronizers.hs

@@ -11,9 +11,9 @@ import           Startlude               hiding ( check
 import qualified Startlude.ByteStream          as ByteStream
 import qualified Startlude.ByteStream.Char8    as ByteStream
 
+import           Control.Carrier.Lift           ( runM )
 import qualified Control.Effect.Reader.Labelled
                                                as Fused
-import           Control.Carrier.Lift           ( runM )
 import           Control.Monad.Trans.Reader     ( mapReaderT )
 import           Control.Monad.Trans.Resource
 import           Data.Attoparsec.Text
@@ -21,50 +21,57 @@ import qualified Data.ByteString               as BS
 import qualified Data.ByteString.Char8         as B8
 import qualified Data.Conduit                  as Conduit
 import qualified Data.Conduit.Combinators      as Conduit
-import qualified Data.Conduit.Tar              as Conduit
 import           Data.Conduit.Shell      hiding ( arch
-                                                , patch
-                                                , stream
                                                 , hostname
+                                                , patch
+                                                , split
+                                                , stream
                                                 )
+import qualified Data.Conduit.Tar              as Conduit
 import           Data.FileEmbed
 import qualified Data.HashMap.Strict           as HM
 import           Data.IORef
 import           Data.String.Interpolate.IsString
 import qualified Data.Yaml                     as Yaml
 import           Exinst
-import           System.FilePath                ( splitPath
+import qualified Streaming.Conduit             as Conduit
+import qualified Streaming.Prelude             as Stream
+import qualified Streaming.Zip                 as Stream
+import           System.Directory
+import           System.FilePath                ( (</>)
                                                 , joinPath
-                                                , (</>)
+                                                , splitPath
                                                 )
 import           System.FilePath.Posix          ( takeDirectory )
-import           System.Directory
 import           System.IO.Error
 import           System.Posix.Files
-import qualified Streaming.Prelude             as Stream
+import           System.Process                 ( callCommand )
-import qualified Streaming.Conduit             as Conduit
-import qualified Streaming.Zip                 as Stream
 
 import           Constants
+import           Control.Effect.Error    hiding ( run )
+import           Control.Effect.Labelled        ( runLabelled )
+import           Daemon.ZeroConf                ( getStart9AgentHostname )
+import           Data.ByteString.Char8          ( split )
+import qualified Data.ByteString.Char8         as C8
+import           Data.Conduit.List              ( consume )
+import qualified Data.Text                     as T
 import           Foundation
+import           Handler.Network
+import qualified Lib.Algebra.Domain.AppMgr     as AppMgr2
 import           Lib.ClientManifest
 import           Lib.Error
 import qualified Lib.External.AppMgr           as AppMgr
 import           Lib.External.Registry
 import           Lib.Sound
 import           Lib.Ssl
-import           Lib.Tor
-import           Lib.Types.Core
-import           Lib.Types.NetAddress
-import           Lib.Types.Emver
 import           Lib.SystemCtl
 import           Lib.SystemPaths         hiding ( (</>) )
+import           Lib.Tor
+import           Lib.Types.Core
+import           Lib.Types.Emver
+import           Lib.Types.NetAddress
 import           Settings
 import           Util.File
-import qualified Lib.Algebra.Domain.AppMgr     as AppMgr2
-import           Daemon.ZeroConf                ( getStart9AgentHostname )
-import qualified Data.Text                     as T
-import           Control.Effect.Error    hiding ( run )
 
 
 data Synchronizer = Synchronizer
@@ -95,15 +102,16 @@ parseKernelVersion = do
     pure $ KernelVersion (Version (major', minor', patch', 0)) arch
 
 synchronizer :: Synchronizer
-synchronizer = sync_0_2_6
+synchronizer = sync_0_2_16
 {-# INLINE synchronizer #-}
 
-sync_0_2_6 :: Synchronizer
+sync_0_2_16 :: Synchronizer
-sync_0_2_6 = Synchronizer
+sync_0_2_16 = Synchronizer
-    "0.2.6"
+    "0.2.16"
     [ syncCreateAgentTmp
     , syncCreateSshDir
     , syncRemoveAvahiSystemdDependency
+    , syncInstallLibAvahi
     , syncInstallAppMgr
     , syncFullUpgrade
     , sync32BitKernel
@@ -112,6 +120,7 @@ sync_0_2_6 = Synchronizer
     , syncInstallDuplicity
     , syncInstallExfatFuse
     , syncInstallExfatUtils
+    , syncUpgradeTor
     , syncInstallAmbassadorUI
     , syncOpenHttpPorts
     , syncUpgradeLifeline
@@ -119,6 +128,10 @@ sync_0_2_6 = Synchronizer
     , syncPrepSslIntermediateCaDir
     , syncPersistLogs
     , syncConvertEcdsaCerts
+    , syncRestarterService
+    , syncInstallEject
+    , syncDropCertificateUniqueness
+    , syncRemoveDefaultNginxCfg
     ]
 
 syncCreateAgentTmp :: SyncOp
@@ -167,8 +180,8 @@ syncFullUpgrade = SyncOp "Full Upgrade" check migrate True
                 Just (Done _ (KernelVersion (Version av) _)) -> if av < (4, 19, 118, 0) then pure True else pure False
                 _ -> pure False
         migrate = liftIO . run $ do
-            shell "apt update"
+            shell "apt-get update --allow-releaseinfo-change"
-            shell "apt full-upgrade -y"
+            shell "apt-get full-upgrade -y"
 
 sync32BitKernel :: SyncOp
 sync32BitKernel = SyncOp "32 Bit Kernel Switch" check migrate True
@@ -192,16 +205,24 @@ syncInstallNginx = SyncOp "Install Nginx" check migrate False
     where
         check   = liftIO . run $ fmap isNothing (shell [i|which nginx || true|] $| conduit await)
         migrate = liftIO . run $ do
-            apt "update"
+            shell "apt-get update --allow-releaseinfo-change"
-            apt "install" "nginx" "-y"
+            shell "apt-get install nginx -y"
+
+syncInstallEject :: SyncOp
+syncInstallEject = SyncOp "Install Eject" check migrate False
+    where
+        check   = liftIO . run $ fmap isNothing (shell [i|which eject || true|] $| conduit await)
+        migrate = liftIO . run $ do
+            shell "apt-get update --allow-releaseinfo-change"
+            shell "apt-get install eject -y"
 
 syncInstallDuplicity :: SyncOp
 syncInstallDuplicity = SyncOp "Install duplicity" check migrate False
     where
         check   = liftIO . run $ fmap isNothing (shell [i|which duplicity || true|] $| conduit await)
         migrate = liftIO . run $ do
-            apt "update"
+            shell "apt-get update --allow-releaseinfo-change"
-            apt "install" "-y" "duplicity"
+            shell "apt-get install -y duplicity"
 
 syncInstallExfatFuse :: SyncOp
 syncInstallExfatFuse = SyncOp "Install exfat-fuse" check migrate False
@@ -213,8 +234,8 @@ syncInstallExfatFuse = SyncOp "Install exfat-fuse" check migrate False
                             ProcessException _ (ExitFailure 1) -> pure True
                             _ -> throwIO e
         migrate = liftIO . run $ do
-            apt "update"
+            shell "apt-get update --allow-releaseinfo-change"
-            apt "install" "-y" "exfat-fuse"
+            shell "apt-get install -y exfat-fuse"
 
 syncInstallExfatUtils :: SyncOp
 syncInstallExfatUtils = SyncOp "Install exfat-utils" check migrate False
@@ -226,8 +247,21 @@ syncInstallExfatUtils = SyncOp "Install exfat-utils" check migrate False
                             ProcessException _ (ExitFailure 1) -> pure True
                             _ -> throwIO e
         migrate = liftIO . run $ do
-            apt "update"
+            shell "apt-get update --allow-releaseinfo-change"
-            apt "install" "-y" "exfat-utils"
+            shell "apt-get install -y exfat-utils"
+
+syncInstallLibAvahi :: SyncOp
+syncInstallLibAvahi = SyncOp "Install libavahi-client" check migrate False
+    where
+        check =
+            liftIO
+                $       (run (shell [i|dpkg -l|] $| shell [i|grep libavahi-client3|] $| conduit await) $> False)
+                `catch` \(e :: ProcessException) -> case e of
+                            ProcessException _ (ExitFailure 1) -> pure True
+                            _ -> throwIO e
+        migrate = liftIO . run $ do
+            shell "apt-get update --allow-releaseinfo-change"
+            shell "apt-get install -y libavahi-client3"
 
 syncWriteConf :: Text -> ByteString -> SystemPath -> SyncOp
 syncWriteConf name contents' confLocation = SyncOp [i|Write #{name} Conf|] check migrate False
@@ -409,9 +443,11 @@ syncInstallAppMgr = SyncOp "Install AppMgr" check migrate False
             Left  _ -> pure True
             Right v -> not . (v <||) <$> asks (appMgrVersionSpec . appSettings)
         migrate = fmap (either absurd id) . runExceptT . flip catchE failUpdate $ do
+            lan <- asks appLanThread
             avs <- asks $ appMgrVersionSpec . appSettings
             av  <- AppMgr.installNewAppMgr avs
             unless (av <|| avs) $ throwE $ AppMgrVersionE av avs
+            flip runReaderT lan $ runLabelled @"lanThread" $ postResetLanLogic -- to accommodate 0.2.x -> 0.2.9 where previous appmgr didn't correctly set up lan
 
 syncUpgradeLifeline :: SyncOp
 syncUpgradeLifeline = SyncOp "Upgrade Lifeline" check migrate False
@@ -469,7 +505,7 @@ replaceDerivativeCerts :: (HasFilesystemBase sig m, Fused.Has (Error S9Error) si
 replaceDerivativeCerts = do
     sid <- getStart9AgentHostname
     let hostname = sid <> ".local"
-    tor            <- getAgentHiddenServiceUrl
+    torAddr        <- getAgentHiddenServiceUrl
 
     caKeyPath      <- toS <$> getAbsoluteLocationFor rootCaKeyPath
     caConfPath     <- toS <$> getAbsoluteLocationFor rootCaOpenSslConfPath
@@ -520,7 +556,7 @@ replaceDerivativeCerts = do
                           , duration          = 365
                           }
         hostname
-        tor
+        torAddr
     liftIO $ do
         putStrLn @Text "openssl logs"
         putStrLn @Text "exit code: "
@@ -536,6 +572,83 @@ replaceDerivativeCerts = do
     liftIO $ renameDirectory sslDirTmp sslDir
     liftIO $ systemCtl RestartService "nginx" $> ()
 
+syncRestarterService :: SyncOp
+syncRestarterService = SyncOp "Install Restarter Service" check migrate True
+    where
+        wantedService = $(embedFile "config/restarter.service")
+        wantedTimer   = $(embedFile "config/restarter.timer")
+        check         = do
+            base <- asks $ appFilesystemBase . appSettings
+            liftIO $ not <$> doesPathExist
+                (toS $ "/etc/systemd/system/timers.target.wants/restarter.timer" `relativeTo` base)
+        migrate = do
+            base <- asks $ appFilesystemBase . appSettings
+            liftIO $ BS.writeFile (toS $ "/etc/systemd/system/restarter.service" `relativeTo` base) wantedService
+            liftIO $ BS.writeFile (toS $ "/etc/systemd/system/restarter.timer" `relativeTo` base) wantedTimer
+            liftIO $ callCommand "systemctl enable restarter.service"
+            liftIO $ callCommand "systemctl enable restarter.timer"
+
+syncUpgradeTor :: SyncOp
+syncUpgradeTor = SyncOp "Install Latest Tor" check migrate False
+    where
+        check = run $ do
+            mTorVersion <- (shell "dpkg -s tor" $| shell "grep '^Version'" $| shell "cut -d ' ' -f2" $| conduit await)
+            let torVersion = case mTorVersion of
+                    Nothing -> panic "invalid output from dpkg, can't read tor version"
+                    Just x  -> x
+            pure $ compareTorVersions torVersion "0.3.5.15-1" == LT
+        migrate = liftIO . run $ do
+            shell "apt-get update --allow-releaseinfo-change"
+            availVersions <-
+                (shell "apt-cache madison tor" $| shell "cut -d '|' -f2" $| shell "xargs" $| conduit consume)
+            latest <- case lastMay $ sortBy compareTorVersions availVersions of
+                Nothing -> throwIO $ ErrorCall "No available versions of tor"
+                Just x  -> pure x
+            shell $ "apt-get install -y tor=" <> if "0.3.5.15-1" `elem` availVersions
+                then "0.3.5.15-1"
+                else (C8.unpack latest)
+        compareTorVersions :: ByteString -> ByteString -> Ordering
+        compareTorVersions a b =
+            let a' = (traverse (readMaybe @Int . decodeUtf8) . (split '.' <=< split '-') $ a)
+                b' = (traverse (readMaybe @Int . decodeUtf8) . (split '.' <=< split '-') $ b)
+            in  case liftA2 compare a' b' of
+                    Nothing -> panic "invalid tor version string"
+                    Just x  -> x
+
+
+syncDropCertificateUniqueness :: SyncOp
+syncDropCertificateUniqueness = SyncOp "Eliminate OpenSSL unique_subject=yes" check migrate False
+    where
+        uni   = "unique_subject = no\n"
+        check = do
+            base         <- asks $ appFilesystemBase . appSettings
+            contentsRoot <-
+                liftIO
+                $       (fmap Just . BS.readFile . toS $ (rootCaDirectory <> "index.txt.attr") `relativeTo` base)
+                `catch` \(e :: IOException) -> if isDoesNotExistError e then pure Nothing else throwIO e
+            contentsInt <-
+                liftIO
+                $ (fmap Just . BS.readFile . toS $ (intermediateCaDirectory <> "index.txt.attr") `relativeTo` base)
+                `catch` \(e :: IOException) -> if isDoesNotExistError e then pure Nothing else throwIO e
+            case (contentsRoot, contentsInt) of
+                (Just root, Just int) -> pure $ uni /= root || uni /= int
+                _                     -> pure True
+        migrate = do
+            base <- asks $ appFilesystemBase . appSettings
+            liftIO $ BS.writeFile (toS $ (rootCaDirectory <> "index.txt.attr") `relativeTo` base) uni
+            liftIO $ BS.writeFile (toS $ (intermediateCaDirectory <> "index.txt.attr") `relativeTo` base) uni
+
+syncRemoveDefaultNginxCfg :: SyncOp
+syncRemoveDefaultNginxCfg = SyncOp "Remove Default Nginx Configuration" check migrate False
+    where
+        check = do
+            base <- asks $ appFilesystemBase . appSettings
+            liftIO $ doesPathExist (toS $ nginxSitesEnabled "default" `relativeTo` base)
+        migrate = do
+            base <- asks $ appFilesystemBase . appSettings
+            liftIO $ removeFileIfExists (toS $ nginxSitesEnabled "default" `relativeTo` base)
+            liftIO $ systemCtl RestartService "nginx" $> ()
+
 failUpdate :: S9Error -> ExceptT Void (ReaderT AgentCtx IO) ()
 failUpdate e = do
     ref <- asks appIsUpdateFailed

agent/src/Lib/SystemPaths.hs

@@ -80,6 +80,11 @@ readSystemPath path = do
         $       (Just <$> readFile (toS loadPath))
         `catch` (\(e :: IOException) -> if isDoesNotExistError e then pure Nothing else throwIO e)
 
+existsSystemPath :: (HasFilesystemBase sig m, MonadIO m) => SystemPath -> m Bool
+existsSystemPath path = do
+    checkPath <- getAbsoluteLocationFor path
+    liftIO . doesPathExist $ toS checkPath
+
 -- like the above, but throws IO error if file not found
 readSystemPath' :: (HasFilesystemBase sig m, MonadIO m) => SystemPath -> m Text
 readSystemPath' path = do
@@ -188,6 +193,9 @@ agentTorHiddenServicePrivateKeyPath = agentTorHiddenServiceDirectory <> "/hs_ed2
 serverNamePath :: SystemPath
 serverNamePath = "/root/agent/name.txt"
 
+disableAutoCheckUpdatesPath :: SystemPath
+disableAutoCheckUpdatesPath = "/root/agent/.disableAutoCheckUpdates"
+
 altRegistryUrlPath :: SystemPath
 altRegistryUrlPath = "/root/agent/alt_registry_url.txt"
 

agent/src/Lib/Tor.hs

@@ -3,11 +3,20 @@ module Lib.Tor where
 import           Startlude
 
 import qualified Data.Text                     as T
+import           Network.HTTP.Client
+import           Network.Connection
 
 import           Lib.SystemPaths
+import           Network.HTTP.Client.TLS        ( mkManagerSettings )
+import           Data.Default
 
 getAgentHiddenServiceUrl :: (HasFilesystemBase sig m, MonadIO m) => m Text
 getAgentHiddenServiceUrl = T.strip <$> readSystemPath' agentTorHiddenServiceHostnamePath
 
 getAgentHiddenServiceUrlMaybe :: (HasFilesystemBase sig m, MonadIO m) => m (Maybe Text)
 getAgentHiddenServiceUrlMaybe = fmap T.strip <$> readSystemPath agentTorHiddenServiceHostnamePath
+
+-- | 'newTorManager' currently assumes the tor client lives on the localhost. The port comes in over an argument.
+-- If this is insufficient in the future, feel free to parameterize the host.
+newTorManager :: Word16 -> IO Manager
+newTorManager = newManager . mkManagerSettings def . Just . SockSettingsSimple "127.0.0.1" . fromIntegral

agent/src/Lib/Types/Emver.hs

@@ -56,6 +56,10 @@ instance Show Version where
         let postfix = if q == 0 then "" else '.' : show q in show x <> "." <> show y <> "." <> show z <> postfix
 instance IsString Version where
     fromString s = either error id $ Atto.parseOnly parseVersion (T.pack s)
+instance Read Version where
+    readsPrec _ s = case Atto.parseOnly parseVersion (T.pack s) of
+        Left _ -> []
+        Right a -> [(a, "")]
 
 -- | A change in the value found at 'major' implies a breaking change in the API that this version number describes
 major :: Version -> Word

agent/src/Lib/Types/Emver/Orphans.hs

@@ -3,16 +3,17 @@ module Lib.Types.Emver.Orphans where
 
 import           Startlude
 
+import           Control.Monad.Fail
 import           Data.Aeson
-
+import qualified Data.Attoparsec.Text          as Atto
-import           Lib.Types.Emver
+import qualified Data.Text                     as T
 import           Database.Persist
 import           Database.Persist.Sql
-import qualified Data.Attoparsec.Text          as Atto
+import           Web.HttpApiData
-import           Control.Monad.Fail
-import qualified Data.Text                     as T
 import           Yesod.Core.Dispatch
 
+import           Lib.Types.Emver
+
 instance ToJSON Version where
     toJSON = String . show
 instance FromJSON Version where
@@ -31,9 +32,16 @@ instance FromJSON VersionRange where
 instance PersistField Version where
     toPersistValue   = toPersistValue @Text . show
     fromPersistValue = first T.pack . Atto.parseOnly parseVersion <=< fromPersistValue
-
 instance PersistFieldSql Version where
     sqlType _ = SqlString
+instance FromHttpApiData Version where
+    parseUrlPiece = first toS . Atto.parseOnly parseVersion
+instance ToHttpApiData Version where
+    toUrlPiece = show
+
+instance PathPiece Version where
+    toPathPiece   = show
+    fromPathPiece = hush . Atto.parseOnly parseVersion
 
 instance PathPiece VersionRange where
     toPathPiece   = show

agent/src/Lib/Types/NetAddress.hs

@@ -7,6 +7,10 @@ newtype TorAddress = TorAddress { unTorAddress :: Text } deriving (Eq)
 instance Show TorAddress where
     show = toS . unTorAddress
 
+newtype LanAddress = LanAddress { unLanAddress :: Text } deriving (Eq)
+instance Show LanAddress where
+    show = toS . unLanAddress
+
 newtype LanIp = LanIp { unLanIp :: Text } deriving (Eq)
 instance Show LanIp where
     show = toS . unLanIp

agent/src/Lib/Types/ServerApp.hs

@@ -20,12 +20,14 @@ data StoreApp = StoreApp
     , storeAppDescriptionLong  :: Text
     , storeAppIconUrl          :: Text
     , storeAppVersions         :: NonEmpty StoreAppVersionInfo
+    , storeAppTimestamp        :: UTCTime
     }
     deriving (Eq, Show)
 
 data StoreAppVersionInfo = StoreAppVersionInfo
     { storeAppVersionInfoVersion      :: Version
     , storeAppVersionInfoReleaseNotes :: Text
+    , storeAppVersionInfoInstallAlert :: Maybe Text
     }
     deriving (Eq, Show)
 instance Ord StoreAppVersionInfo where
@@ -34,6 +36,7 @@ instance FromJSON StoreAppVersionInfo where
     parseJSON = withObject "Store App Version Info" $ \o -> do
         storeAppVersionInfoVersion      <- o .: "version"
         storeAppVersionInfoReleaseNotes <- o .: "release-notes"
+        storeAppVersionInfoInstallAlert <- o .:? "install-alert"
         pure StoreAppVersionInfo { .. }
 instance ToJSON StoreAppVersionInfo where
     toJSON StoreAppVersionInfo {..} =

agent/src/Lib/WebServer.hs

@@ -45,6 +45,7 @@ import           Handler.Backups
 import           Handler.Hosts
 import           Handler.Icons
 import           Handler.Login
+import           Handler.Network
 import           Handler.Notifications
 import           Handler.PasswordUpdate
 import           Handler.PowerOff
@@ -55,6 +56,8 @@ import           Handler.Status
 import           Handler.Wifi
 import           Handler.V0
 import           Settings
+import           Network.HTTP.Types.Header ( hOrigin )
+import           Data.List (lookup)
 
 -- This line actually creates our YesodDispatch instance. It is the second half
 -- of the call to mkYesodData which occurs in Foundation.hs. Please see the
@@ -64,20 +67,11 @@ mkYesodDispatch "AgentCtx" resourcesAgentCtx
 instance YesodSubDispatch Auth AgentCtx where
     yesodSubDispatch = $(mkYesodSubDispatch resourcesAuth)
 
--- | Convert our foundation to a WAI Application by calling @toWaiAppPlain@ and
+dynamicCorsResourcePolicy :: Request -> Maybe CorsResourcePolicy
--- applying some additional middlewares.
+dynamicCorsResourcePolicy req = Just . policy . lookup hOrigin $ requestHeaders req
-makeApplication :: AgentCtx -> IO Application
-makeApplication foundation = do
-    logWare <- makeLogWare foundation
-    -- Create the WAI application and apply middlewares
-    appPlain <- toWaiAppPlain foundation
-    let origin = case appCorsOverrideStar $ appSettings foundation of
-            Nothing -> Nothing
-            Just override -> Just ([encodeUtf8 override], True)
-    pure . logWare . cors (const . Just $ policy origin) . defaultMiddlewaresNoLogging $ appPlain
     where
         policy o = simpleCorsResourcePolicy
-            { corsOrigins        = o
+            { corsOrigins        = (\o' -> ([o'], True)) <$> o
             , corsMethods        = ["GET", "POST", "HEAD", "PUT", "DELETE", "TRACE", "CONNECT", "OPTIONS", "PATCH"]
             , corsRequestHeaders = [ "app-version"
                                    , "Accept"
@@ -138,6 +132,15 @@ makeApplication foundation = do
             , corsIgnoreFailures = True
             }
 
+-- | Convert our foundation to a WAI Application by calling @toWaiAppPlain@ and
+-- applying some additional middlewares.
+makeApplication :: AgentCtx -> IO Application
+makeApplication foundation = do
+    logWare <- makeLogWare foundation
+    -- Create the WAI application and apply middlewares
+    appPlain <- toWaiAppPlain foundation
+    pure . logWare . cors dynamicCorsResourcePolicy . defaultMiddlewaresNoLogging $ appPlain
+
 startWeb :: AgentCtx -> IO ()
 startWeb foundation = do
     app <- makeApplication foundation

agent/src/Model.hs

@@ -59,4 +59,7 @@ BackupRecord sql=backup
 IconDigest
     Id AppId
     tag (Digest MD5)
+
+WelcomeAck
+    Id Version
 |]

agent/src/Settings.hs

@@ -41,7 +41,9 @@ data AppSettings = AppSettings
     -- ^ Should all log messages be displayed?
     , appMgrVersionSpec         :: VersionRange
     , appFilesystemBase         :: Text
-    , appCorsOverrideStar       :: Maybe Text
+    , appTorSocksPort           :: Word16
+    -- ^ Port on localhost where the tor client is listening, defaults to 9050
+    , appTorRestartCooldown     :: NominalDiffTime
     }
     deriving Show
 
@@ -64,7 +66,8 @@ instance FromJSON AppSettings where
 
         appMgrVersionSpec         <- o .: "app-mgr-version-spec"
         appFilesystemBase         <- o .: "filesystem-base"
-        appCorsOverrideStar       <- o .:? "cors-override-star"
+        appTorSocksPort           <- o .:? "tor-socks-port" .!= 9050
+        appTorRestartCooldown     <- o .:? "tor-restart-cooldown" .!= (secondsToNominalDiffTime 600)
         return AppSettings { .. }
 
 -- | Raw bytes at compile time of @config/settings.yml@

agent/src/Startlude.hs

@@ -69,12 +69,12 @@ instance MonadResource m => MonadResource (FE.ReaderC r m)  where
 instance MonadResource m => MonadResource (FE.ErrorC e m) where
     liftResourceT = lift . liftResourceT
 
-
 instance MonadThrow (sub m) => MonadThrow (FE.Labelled label sub m) where
     throwM = FE.Labelled . throwM
 instance MonadThrow m => MonadThrow (FE.LiftC m) where
     throwM = FE.LiftC . throwM
 
+instance MonadLogger m => MonadLogger (FE.ErrorC e m) where
 instance MonadLogger m => MonadLogger (FE.LiftC m) where
 instance MonadLogger (sub m) => MonadLogger (FE.Labelled label sub m) where
     monadLoggerLog a b c d = FE.Labelled $ monadLoggerLog a b c d
@@ -91,6 +91,13 @@ instance MonadHandler (sub m) => MonadHandler (FE.Labelled label sub m) where
     liftHandler    = FE.Labelled . liftHandler
     liftSubHandler = FE.Labelled . liftSubHandler
 
+
+instance MonadHandler m => MonadHandler (FE.ErrorC e m) where
+    type HandlerSite (FE.ErrorC e m) = HandlerSite m
+    type SubHandlerSite (FE.ErrorC e m) = SubHandlerSite m
+    liftHandler    = lift . liftHandler
+    liftSubHandler = lift . liftSubHandler
+
 instance MonadTransControl t => MonadTransControl (FE.Labelled k t) where
     type StT (FE.Labelled k t) a = StT t a
     liftWith f = FE.Labelled $ liftWith $ \run -> f (run . FE.runLabelled)

agent/src/Startlude/ByteStream.hs

@@ -1,13 +1,7 @@
 -- {-# OPTIONS_GHC -fno-warn-unused-imports #-}
 module Startlude.ByteStream
-    ( module Startlude.ByteStream
+    ( module BS
-    , module BS
+    ) where
-    )
-where
 
-import           Data.ByteString.Streaming     as BS
+import           Streaming.ByteString          as BS
                                          hiding ( ByteString )
-import           Data.ByteString.Streaming     as X
-                                                ( ByteString )
-
-type ByteStream m = X.ByteString m

agent/src/Startlude/ByteStream/Char8.hs

@@ -1,7 +1,5 @@
 module Startlude.ByteStream.Char8
     ( module X
-    )
+    ) where
-where
 
-import           Data.ByteString.Streaming.Char8
+import           Streaming.ByteString.Char8    as X
-                                               as X

agent/stack.yaml

@@ -1,10 +1,10 @@
-resolver: nightly-2020-09-29
+resolver: lts-17.10
 
 packages:
-- .
+  - .
 
 extra-deps:
-  - aeson-1.4.7.1
+  # - aeson-1.4.7.1
   - aeson-flatten-0.1.0.2
   - exinst-0.8
   - fused-effects-1.1.0.0
@@ -12,13 +12,14 @@ extra-deps:
   - git-embed-0.1.0
   - json-stream-0.4.2.4
   - protolude-0.3.0
+  - streaming-bytestring-0.1.7
   - streaming-conduit-0.1.2.2
   - streaming-utils-0.2.0.0
   # to avoid the ridiculous bug where stat64 is not found (only affects development)
-  - git: https://github.com/ProofOfKeags/persistent.git
+  # - git: https://github.com/ProofOfKeags/persistent.git
-    commit: 3b52b13d9ce79cdef14bb1c37cc527657a529462
+  # commit: 3b52b13d9ce79cdef14bb1c37cc527657a529462
-    subdirs:
+  # subdirs:
-      - persistent-sqlite
+  # - persistent-sqlite
 
 ghc-options:
   "$locals": -fwrite-ide-info

agent/test/Lib/External/AppManifestSpec.hs

@@ -66,12 +66,65 @@ assets:
 hidden-service-version: v3
 |]
 
+mastodon330Manifest :: ByteString
+mastodon330Manifest = [i|
+---
+id: mastodon
+version: 3.3.0.1
+title: Mastodon
+description:
+  short: "A free, open-source social network server."
+  long: "Mastodon is a free, open-source social network server based on ActivityPub where users can follow friends and discover new ones. On Mastodon, users can publish anything they want: links, pictures, text, video. All Mastodon servers are interoperable as a federated network (users on one server can seamlessly communicate with users from another one, including non-Mastodon software that implements ActivityPub)!"
+release-notes: Added an acation to reset the admin password
+install-alert: "After starting mastodon for the first time, it can take a long time (several minutes) to be ready.\nPlease be patient. On future starts of the service, it will be faster, but still takes longer than other services.\nMake sure to sign up for a user before giving out your link. The first user to sign up is set as the admin user.\n"
+uninstall-alert: ~
+restore-alert: ~
+start-alert: "It may take several minutes after startup for this service to be ready for use.\n"
+has-instructions: true
+os-version-required: ">=0.2.8"
+os-version-recommended: ">=0.2.8"
+ports:
+  - internal: 80
+    tor: 80
+    lan: standard
+  - internal: 443
+    tor: 443
+    lan:
+      custom:
+        port: 443
+  - internal: 3000
+    tor: 3000
+    lan: ~
+  - internal: 4000
+    tor: 4000
+    lan: ~
+image:
+  type: tar
+shm-size-mb: ~
+mount: /root/persistence
+public: ~
+shared: ~
+assets: []
+hidden-service-version: v3
+dependencies: {}
+actions:
+  - id: reset-admin-password
+    name: Reset Admin Password
+    description: This action will reset your admin password to a random value
+    allowed-statuses:
+      - RUNNING
+    command:
+      - docker_entrypoint.sh
+      - reset_admin_password.sh
+|]
+
+
 spec :: Spec
 spec = do
-    describe "parsing app manifest ports" $ do
+  describe "parsing app manifest ports" $ do
-        it "should yield true for cups 0.2.3" $ do
+    it "should parse mastodon 3.3.0" $ do
-            res <- decodeThrow @IO @(AppManifest 0) cups023Manifest
+      res <- decodeThrow @IO @AppManifest mastodon330Manifest
-            uiAvailable res `shouldBe` True
+      print res
-        it "should yield false for cups 0.2.3 Mod" $ do
+      lanUiAvailable res `shouldBe` True
-            res <- decodeThrow @IO @(AppManifest 0) cups023ManifestModNoUI
+      torUiAvailable res `shouldBe` True
-            uiAvailable res `shouldBe` False
+

appmgr/.github/workflows/rust.yml

@@ -1,30 +0,0 @@
-name: Rust
-
-on: [push]
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v1
-      - name: Cache .cargo
-        uses: actions/cache@v1
-        with:
-          path: ~/.cargo
-          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
-            ${{ runner.os }}-cargo-
-      - name: Cache target release directory
-        uses: actions/cache@v1
-        with:
-          path: target/release
-          key: ${{ runner.os }}-target-release-${{ hashFiles('**/Cargo.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-target-release-${{ hashFiles('**/Cargo.lock') }}
-            ${{ runner.os }}-target-release-
-      - name: Check
-        run: cargo check
-      - name: Test
-        run: cargo test --release

appmgr/.gitignore

@@ -1,2 +1,3 @@
 /target
 **/*.rs.bk
+.DS_Store

appmgr/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
-name = "appmgr"
-version = "0.2.6"
 authors = ["Aiden McClelland <me@drbonez.dev>"]
 edition = "2018"
+name = "appmgr"
+version = "0.2.16"
 
 [lib]
 name = "appmgrlib"
@@ -13,37 +13,48 @@ name = "appmgr"
 path = "src/main.rs"
 
 [features]
-default = []
+avahi = ["avahi-sys"]
+default = ["avahi"]
 portable = []
 production = []
 
 [dependencies]
-emver = { git = "https://github.com/Start9Labs/emver-rs.git", version = "0.1.0", features = ["serde"] }
+async-trait = "0.1.42"
-argonautica = "0.2.0"
+avahi-sys = { git = "https://github.com/Start9Labs/avahi-sys", branch = "feature/dynamic-linking", features = [
-async-trait = "0.1.41"
+  "dynamic",
+], optional = true }
 base32 = "0.4.0"
 clap = "2.33"
+ctrlc = "3.1.7"
 ed25519-dalek = "1.0.1"
+emver = { version = "0.1.0", features = ["serde"] }
 failure = "0.1.8"
 file-lock = "1.1"
-futures = "0.3.7"
+futures = "0.3.8"
 git-version = "0.3.4"
+http = "0.2.3"
 itertools = "0.9.0"
 lazy_static = "1.4"
+libc = "0.2.86"
 linear-map = { version = "1.2", features = ["serde_impl"] }
 log = "0.4.11"
+nix = "0.19.1"
 openssl = "0.10.30"
 pest = "2.1"
 pest_derive = "2.1"
 prettytable-rs = "0.8.0"
 rand = "0.7.3"
 regex = "1.4.2"
-reqwest = { version = "0.10.8", features = ["stream", "json"] }
+reqwest = { version = "0.10.9", features = ["stream", "json"] }
 rpassword = "5.0.0"
-serde = { version = "1.0.117", features = ["derive", "rc"] }
+rust-argon2 = "0.8.3"
-serde_yaml = "0.8.14"
+scopeguard = "1.1" # because avahi-sys fucks your shit up
+serde = { version = "1.0.118", features = ["derive", "rc"] }
 serde_cbor = "0.11.1"
 serde_json = "1.0.59"
+serde_yaml = "0.8.14"
 simple-logging = "2.0"
-tokio = { version = "0.2.22", features = ["full"] }
+tokio = { version = "0.3.5", features = ["full"] }
-tokio-tar = "0.2.0"
+tokio-compat-02 = "0.1.2"
+tokio-tar = { version = "0.3.0", git = "https://github.com/dr-bonez/tokio-tar.git", rev = "1ba710f3" }
+yajrc = { version = "0.1.0", git = "https://github.com/dr-bonez/yajrc", rev = "c2952a4a21c50f7be6f8003afa37ee77deb66d56" }

appmgr/README.md

@@ -1,5 +1,17 @@
 # appmgr
 
+# Instructions
+
+Clone the repo and enter the appmgr directory
+
+`git clone https://github.com/Start9Labs/embassy-os.git`
+
+`cd embassy-os/appmgr`
+
+Install the portable version of appmgr
+
+`cargo install --path=. --features=portable --no-default-features`
+
 ## Exit Codes
 1. General Error
 2. File System IO Error
@@ -7,4 +19,4 @@
 4. Config Spec violation
 5. Config Rules violation
 6. Requested value does not exist
-7. Invalid Backup Password
+7. Invalid Backup Password

appmgr/build-dev.sh

@@ -3,7 +3,12 @@
 set -e
 shopt -s expand_aliases
 
+if [ "$0" != "./build-dev.sh" ]; then
+	>&2 echo "Must be run from appmgr directory"
+	exit 1
+fi
+
 alias 'rust-arm-builder'='docker run --rm -it -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:latest'
 
 cd ..
-rust-arm-builder sh -c "(cd appmgr && cargo build --release)"
+rust-arm-builder sh -c "(cd appmgr && cargo build)"

appmgr/build-portable.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -e
+shopt -s expand_aliases
+
+if [ "$0" != "./build-portable.sh" ]; then
+	>&2 echo "Must be run from appmgr directory"
+	exit 1
+fi
+
+alias 'rust-musl-builder'='docker run --rm -it -v "$HOME"/.cargo/registry:/root/.cargo/registry -v "$(pwd)":/home/rust/src messense/rust-musl-cross:x86_64-musl'
+
+cd ..
+rust-musl-builder sh -c "(cd appmgr && cargo build --release --target=x86_64-unknown-linux-musl --features=portable,production --no-default-features)"
+cd appmgr

appmgr/build-prod.sh

@@ -3,6 +3,11 @@
 set -e
 shopt -s expand_aliases
 
+if [ "$0" != "./build-prod.sh" ]; then
+	>&2 echo "Must be run from appmgr directory"
+	exit 1
+fi
+
 alias 'rust-arm-builder'='docker run --rm -it -v "$HOME/.cargo/registry":/root/.cargo/registry -v "$(pwd)":/home/rust/src start9/rust-arm-cross:latest'
 
 cd ..

appmgr/src/actions.rs

@@ -0,0 +1,116 @@
+use std::os::unix::process::ExitStatusExt;
+use std::process::Stdio;
+
+use linear_map::set::LinearSet;
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, Error as IoError};
+use yajrc::RpcError;
+
+use crate::apps::DockerStatus;
+
+pub const STATUS_NOT_ALLOWED: i32 = -2;
+pub const INVALID_COMMAND: i32 = -3;
+
+#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Action {
+    pub id: String,
+    pub name: String,
+    pub description: String,
+    #[serde(default)]
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub warning: Option<String>,
+    pub allowed_statuses: LinearSet<DockerStatus>,
+    pub command: Vec<String>,
+}
+
+async fn tee<R: AsyncRead + Unpin, W: AsyncWrite + Unpin>(
+    mut r: R,
+    mut w: W,
+) -> Result<Vec<u8>, IoError> {
+    let mut res = Vec::new();
+    let mut buf = vec![0; 2048];
+    let mut bytes;
+    while {
+        bytes = r.read(&mut buf).await?;
+        bytes != 0
+    } {
+        res.extend_from_slice(&buf[..bytes]);
+        w.write_all(&buf[..bytes]).await?;
+    }
+    w.flush().await?;
+    Ok(res)
+}
+
+impl Action {
+    pub async fn perform(&self, app_id: &str) -> Result<String, RpcError> {
+        let man = crate::apps::manifest(app_id)
+            .await
+            .map_err(failure::Error::from)
+            .map_err(failure::Error::compat)?;
+        let status = crate::apps::status(app_id, true)
+            .await
+            .map_err(failure::Error::from)
+            .map_err(failure::Error::compat)?
+            .status;
+        if !self.allowed_statuses.contains(&status) {
+            return Err(RpcError {
+                code: STATUS_NOT_ALLOWED,
+                message: format!(
+                    "{} is in status {:?} which is not allowed by {}",
+                    app_id, status, self.id
+                ),
+                data: None,
+            });
+        }
+        let mut cmd = if status == DockerStatus::Running {
+            let mut cmd = tokio::process::Command::new("docker");
+            cmd.arg("exec").arg(&app_id).args(&self.command);
+            cmd
+        } else {
+            let mut cmd = tokio::process::Command::new("docker");
+            let entrypoint = self.command.get(0).ok_or_else(|| RpcError {
+                code: INVALID_COMMAND,
+                message: "Command Cannot Be Empty".to_owned(),
+                data: None,
+            })?;
+            cmd.arg("run")
+                .arg("--rm")
+                .arg("--name")
+                .arg(format!("{}_{}", app_id, self.id))
+                .arg("--mount")
+                .arg(format!(
+                    "type=bind,src={}/{},dst={}",
+                    crate::VOLUMES,
+                    app_id,
+                    man.mount.display()
+                ))
+                .arg("--entrypoint")
+                .arg(entrypoint)
+                .arg(format!("start9/{}", app_id))
+                .args(&self.command[1..]);
+            // TODO: 0.3.0: net, tor, shm
+            cmd
+        };
+        cmd.stdout(Stdio::piped());
+        cmd.stderr(Stdio::piped());
+        let mut child = cmd.spawn()?;
+
+        let (stdout, stderr) = futures::try_join!(
+            tee(child.stdout.take().unwrap(), tokio::io::sink()),
+            tee(child.stderr.take().unwrap(), tokio::io::sink())
+        )?;
+
+        let status = child.wait().await?;
+        if status.success() {
+            String::from_utf8(stdout).map_err(From::from)
+        } else {
+            Err(RpcError {
+                code: status
+                    .code()
+                    .unwrap_or_else(|| status.signal().unwrap_or(0) + 128),
+                message: String::from_utf8(stderr)?,
+                data: None,
+            })
+        }
+    }
+}

appmgr/src/apps.rs

@@ -131,7 +131,7 @@ pub async fn remove(id: &str) -> Result<(), failure::Error> {
     Ok(())
 }
 
-pub async fn status(id: &str) -> Result<AppStatus, Error> {
+pub async fn status(id: &str, remap_crashed: bool) -> Result<AppStatus, Error> {
     let output = std::process::Command::new("docker")
         .args(&["inspect", id, "--format", "{{.State.Status}}"])
         .stdout(std::process::Stdio::piped())
@@ -155,6 +155,19 @@ pub async fn status(id: &str) -> Result<AppStatus, Error> {
             "restarting" => DockerStatus::Restarting,
             "removing" => DockerStatus::Removing,
             "dead" => DockerStatus::Dead,
+            "exited"
+                if remap_crashed && {
+                    let path = PersistencePath::from_ref("running.yaml");
+                    if let Some(mut f) = path.maybe_read(false).await.transpose()? {
+                        let running: Vec<String> = from_yaml_async_reader(&mut *f).await?;
+                        running.iter().filter(|a| a.as_str() == id).next().is_some()
+                    } else {
+                        false
+                    }
+                } =>
+            {
+                DockerStatus::Restarting
+            }
             "created" | "exited" => DockerStatus::Stopped,
             "paused" => DockerStatus::Paused,
             _ => Err(format_err!("unknown status: {}", status))?,
@@ -275,7 +288,7 @@ pub async fn info_full(
     Ok(AppInfoFull {
         info: info(id).await?,
         status: if with_status {
-            Some(status(id).await?)
+            Some(status(id, true).await?)
         } else {
             None
         },
@@ -379,8 +392,12 @@ pub async fn list(
     let info = list_info().await?;
     futures::future::join_all(info.into_iter().map(move |(id, info)| async move {
         let (status, manifest, config, dependencies) = futures::try_join!(
-            OptionFuture::from(if with_status { Some(status(&id)) } else { None })
+            OptionFuture::from(if with_status {
-                .map(Option::transpose),
+                Some(status(&id, true))
+            } else {
+                None
+            })
+            .map(Option::transpose),
             OptionFuture::from(if with_manifest {
                 Some(manifest(&id))
             } else {

appmgr/src/backup.rs

@@ -1,15 +1,28 @@
+use std::os::unix::process::ExitStatusExt;
 use std::path::Path;
 
-use argonautica::{Hasher, Verifier};
+use argon2::Config;
+use emver::Version;
 use futures::try_join;
 use futures::TryStreamExt;
+use rand::Rng;
+use serde::Serialize;
 
-use crate::apps;
 use crate::util::from_yaml_async_reader;
+use crate::util::to_yaml_async_writer;
 use crate::util::Invoke;
+use crate::util::PersistencePath;
+use crate::version::VersionT;
 use crate::Error;
 use crate::ResultExt;
 
+#[derive(Debug, Clone, Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub struct Metadata {
+    pub app_version: Version,
+    pub os_version: &'static Version,
+}
+
 pub async fn create_backup<P: AsRef<Path>>(
     path: P,
     app_id: &str,
@@ -21,6 +34,7 @@ pub async fn create_backup<P: AsRef<Path>>(
         crate::error::FILESYSTEM_ERROR,
         "Backup Path Must Be Directory"
     );
+    let metadata_path = path.join("metadata.yaml");
     let pw_path = path.join("password");
     let data_path = path.join("data");
     let tor_path = path.join("tor");
@@ -35,10 +49,7 @@ pub async fn create_backup<P: AsRef<Path>>(
         let mut hash = String::new();
         f.read_to_string(&mut hash).await?;
         crate::ensure_code!(
-            Verifier::new()
+            argon2::verify_encoded(&hash, password.as_bytes())
-                .with_password(password)
-                .with_hash(hash)
-                .verify()
                 .with_code(crate::error::INVALID_BACKUP_PASSWORD)?,
             crate::error::INVALID_BACKUP_PASSWORD,
             "Invalid Backup Decryption Password"
@@ -47,16 +58,24 @@ pub async fn create_backup<P: AsRef<Path>>(
     {
         // save password
         use tokio::io::AsyncWriteExt;
-
+        let salt = rand::thread_rng().gen::<[u8; 32]>();
-        let mut hasher = Hasher::default();
+        let hash = argon2::hash_encoded(password.as_bytes(), &salt, &Config::default()).unwrap(); // this is safe because apparently the API was poorly designed
-        hasher.opt_out_of_secret_key(true);
-        let hash = hasher.with_password(password).hash().no_code()?;
         let mut f = tokio::fs::File::create(pw_path).await?;
         f.write_all(hash.as_bytes()).await?;
         f.flush().await?;
     }
 
-    let status = crate::apps::status(app_id).await?;
+    let info = crate::apps::info(app_id).await?;
+    to_yaml_async_writer(
+        tokio::fs::File::create(metadata_path).await?,
+        &Metadata {
+            app_version: info.version,
+            os_version: crate::version::Current::new().semver(),
+        },
+    )
+    .await?;
+
+    let status = crate::apps::status(app_id, false).await?;
     let exclude = if volume_path.is_dir() {
         let ignore_path = volume_path.join(".backupignore");
         if ignore_path.is_file() {
@@ -124,6 +143,7 @@ pub async fn restore_backup<P: AsRef<Path>>(
         crate::error::FILESYSTEM_ERROR,
         "Backup Path Must Be Directory"
     );
+    let metadata_path = path.join("metadata.yaml");
     let pw_path = path.join("password");
     let data_path = path.join("data");
     let tor_path = path.join("tor");
@@ -138,17 +158,14 @@ pub async fn restore_backup<P: AsRef<Path>>(
         let mut hash = String::new();
         f.read_to_string(&mut hash).await?;
         crate::ensure_code!(
-            Verifier::new()
+            argon2::verify_encoded(&hash, password.as_bytes())
-                .with_password(password)
-                .with_hash(hash)
-                .verify()
                 .with_code(crate::error::INVALID_BACKUP_PASSWORD)?,
             crate::error::INVALID_BACKUP_PASSWORD,
             "Invalid Backup Decryption Password"
         );
     }
 
-    let status = crate::apps::status(app_id).await?;
+    let status = crate::apps::status(app_id, false).await?;
     let running = status.status == crate::apps::DockerStatus::Running;
     if running {
         crate::control::stop_app(app_id, true, false).await?;
@@ -181,12 +198,21 @@ pub async fn restore_backup<P: AsRef<Path>>(
     );
 
     // Fix the tor address in apps.yaml
-    let mut yhdl = apps::list_info_mut().await?;
+    let mut yhdl = crate::apps::list_info_mut().await?;
     if let Some(app_info) = yhdl.get_mut(app_id) {
         app_info.tor_address = Some(crate::tor::read_tor_address(app_id, None).await?);
     }
     yhdl.commit().await?;
 
+    tokio::fs::copy(
+        metadata_path,
+        Path::new(crate::VOLUMES)
+            .join(app_id)
+            .join("start9")
+            .join("restore.yaml"),
+    )
+    .await?;
+
     // Attempt to configure the service with the config coming from restoration
     let cfg_path = Path::new(crate::VOLUMES)
         .join(app_id)
@@ -200,6 +226,28 @@ pub async fn restore_backup<P: AsRef<Path>>(
     }
 
     crate::tor::restart().await?;
+    // Delete the fullchain certificate, so it can be regenerated with the restored tor pubkey address
+    PersistencePath::from_ref("apps")
+        .join(&app_id)
+        .join("cert-local.fullchain.crt.pem")
+        .delete()
+        .await?;
+    crate::tor::write_lan_services(
+        &crate::tor::services_map(&PersistencePath::from_ref(crate::SERVICES_YAML)).await?,
+    )
+    .await?;
+    let svc_exit = std::process::Command::new("service")
+        .args(&["nginx", "reload"])
+        .status()?;
+    crate::ensure_code!(
+        svc_exit.success(),
+        crate::error::GENERAL_ERROR,
+        "Failed to Reload Nginx: {}",
+        svc_exit
+            .code()
+            .or_else(|| { svc_exit.signal().map(|a| 128 + a) })
+            .unwrap_or(0)
+    );
 
     Ok(())
 }

appmgr/src/cert-local.csr.conf.template

@@ -0,0 +1,10 @@
+[req]
+default_bits = 4096
+default_md = sha256
+distinguished_name = req_distinguished_name
+prompt = no
+
+[req_distinguished_name]
+CN = {hostname}.local
+O = Start9 Labs
+OU = Embassy

appmgr/src/config/mod.rs

@@ -137,7 +137,9 @@ pub async fn configure(
             &mut res.stopped,
         )
         .await?;
-        if crate::apps::status(&dependent).await?.status != crate::apps::DockerStatus::Stopped {
+        if crate::apps::status(&dependent, false).await?.status
+            != crate::apps::DockerStatus::Stopped
+        {
             crate::control::stop_app(&dependent, false, dry_run).await?;
             res.stopped.insert(
                 // TODO: maybe don't do this if its not running
@@ -283,7 +285,8 @@ pub async fn configure(
                 crate::apps::set_configured(name, true).await?;
                 crate::apps::set_recoverable(name, false).await?;
             }
-            if crate::apps::status(name).await?.status != crate::apps::DockerStatus::Stopped {
+            if crate::apps::status(name, false).await?.status != crate::apps::DockerStatus::Stopped
+            {
                 if !dry_run {
                     crate::apps::set_needs_restart(name, true).await?;
                 }

appmgr/src/config/spec.rs

@@ -1864,6 +1864,9 @@ mod test {
             hidden_service_version: crate::tor::HiddenServiceVersion::V3,
             dependencies: deps,
             extra: LinearMap::new(),
+            install_alert: None,
+            restore_alert: None,
+            uninstall_alert: None,
         })
         .unwrap();
         let config = spec

appmgr/src/control.rs

@@ -1,9 +1,10 @@
 use std::path::Path;
 
 use futures::future::{BoxFuture, FutureExt};
-use linear_map::LinearMap;
+use linear_map::{set::LinearSet, LinearMap};
 
 use crate::dependencies::{DependencyError, TaggedDependencyError};
+use crate::util::{from_yaml_async_reader, PersistencePath, YamlUpdateHandle};
 use crate::Error;
 
 pub async fn start_app(name: &str, update_metadata: bool) -> Result<(), Error> {
@@ -19,14 +20,17 @@ pub async fn start_app(name: &str, update_metadata: bool) -> Result<(), Error> {
         true,
     )
     .await?;
-    let status = crate::apps::status(name).await?.status;
+    let status = crate::apps::status(name, false).await?.status;
     if status == crate::apps::DockerStatus::Stopped {
         if update_metadata {
             crate::config::configure(name, None, None, false).await?;
-            crate::dependencies::update_shared(name).await?;
             crate::dependencies::update_binds(name).await?;
         }
         crate::apps::set_needs_restart(name, false).await?;
+        let mut running = YamlUpdateHandle::<LinearSet<String>>::new_or_default(
+            PersistencePath::from_ref("running.yaml"),
+        )
+        .await?;
         let output = tokio::process::Command::new("docker")
             .args(&["start", name])
             .stdout(std::process::Stdio::null())
@@ -38,6 +42,8 @@ pub async fn start_app(name: &str, update_metadata: bool) -> Result<(), Error> {
             "Failed to Start Application: {}",
             std::str::from_utf8(&output.stderr).unwrap_or("Unknown Error")
         );
+        running.insert(name.to_owned());
+        running.commit().await?;
     } else if status == crate::apps::DockerStatus::Paused {
         resume_app(name).await?;
     }
@@ -67,6 +73,10 @@ pub async fn stop_app(
             true,
         )
         .await?;
+        let mut running = YamlUpdateHandle::<LinearSet<String>>::new_or_default(
+            PersistencePath::from_ref("running.yaml"),
+        )
+        .await?;
         log::info!("Stopping {}", name);
         let output = tokio::process::Command::new("docker")
             .args(&["stop", "-t", "25", name])
@@ -79,6 +89,8 @@ pub async fn stop_app(
             "Failed to Stop Application: {}",
             std::str::from_utf8(&output.stderr).unwrap_or("Unknown Error")
         );
+        running.remove(name);
+        running.commit().await?;
         crate::util::unlock(lock).await?;
     }
     Ok(res)
@@ -98,7 +110,7 @@ pub async fn stop_dependents(
     ) -> BoxFuture<'a, Result<(), Error>> {
         async move {
             for dependent in crate::apps::dependents(name, false).await? {
-                if crate::apps::status(&dependent).await?.status
+                if crate::apps::status(&dependent, false).await?.status
                     != crate::apps::DockerStatus::Stopped
                 {
                     stop_dependents_rec(&dependent, dry_run, DependencyError::NotRunning, res)
@@ -192,3 +204,34 @@ pub async fn resume_app(name: &str) -> Result<(), Error> {
     crate::util::unlock(lock).await?;
     Ok(())
 }
+
+pub async fn repair_app_status() -> Result<(), Error> {
+    let mut running_file = PersistencePath::from_ref("running.yaml")
+        .maybe_read(false)
+        .await
+        .transpose()?;
+    let running: Vec<String> = if let Some(f) = running_file.as_mut() {
+        from_yaml_async_reader::<_, &mut tokio::fs::File>(f).await?
+    } else {
+        Vec::new()
+    };
+    for name in running {
+        let lock = crate::util::lock_file(
+            format!(
+                "{}",
+                Path::new(crate::PERSISTENCE_DIR)
+                    .join("apps")
+                    .join(&name)
+                    .join("control.lock")
+                    .display()
+            ),
+            true,
+        )
+        .await?;
+        if crate::apps::status(&name, false).await?.status == crate::apps::DockerStatus::Stopped {
+            start_app(&name, true).await?;
+        }
+        crate::util::unlock(lock).await?;
+    }
+    Ok(())
+}

appmgr/src/dependencies.rs

@@ -115,7 +115,9 @@ impl DepInfo {
         if !errors.is_empty() {
             return Ok(Err(DependencyError::ConfigUnsatisfied(errors)));
         }
-        if crate::apps::status(dependency_id).await?.status != crate::apps::DockerStatus::Running {
+        if crate::apps::status(dependency_id, false).await?.status
+            != crate::apps::DockerStatus::Running
+        {
             return Ok(Err(DependencyError::NotRunning));
         }
         Ok(Ok(()))
@@ -186,31 +188,6 @@ pub async fn auto_configure(
     crate::config::configure(dependency, Some(dependency_config), None, dry_run).await
 }
 
-pub async fn update_shared(dependency_id: &str) -> Result<(), Error> {
-    let dependency_manifest = crate::apps::manifest(dependency_id).await?;
-    if let Some(shared) = dependency_manifest.shared {
-        for dependent_id in &crate::apps::dependents(dependency_id, false).await? {
-            let dependent_manifest = crate::apps::manifest(&dependent_id).await?;
-            if dependent_manifest
-                .dependencies
-                .0
-                .get(dependency_id)
-                .ok_or_else(|| failure::format_err!("failed to index dependent: {}", dependent_id))?
-                .mount_shared
-            {
-                tokio::fs::create_dir_all(
-                    Path::new(crate::VOLUMES)
-                        .join(dependency_id)
-                        .join(&shared)
-                        .join(&dependent_id),
-                )
-                .await?;
-            }
-        }
-    }
-    Ok(())
-}
-
 pub async fn update_binds(dependent_id: &str) -> Result<(), Error> {
     let dependent_manifest = crate::apps::manifest(dependent_id).await?;
     let dependency_manifests = futures::future::try_join_all(
@@ -220,12 +197,19 @@ pub async fn update_binds(dependent_id: &str) -> Result<(), Error> {
             .into_iter()
             .filter(|(_, info)| info.mount_public || info.mount_shared)
             .map(|(id, info)| async {
-                crate::apps::manifest(&id).await.map(|man| (id, info, man))
+                Ok::<_, Error>(if crate::apps::list_info().await?.contains_key(&id) {
+                    let man = crate::apps::manifest(&id).await?;
+                    Some((id, info, man))
+                } else {
+                    None
+                })
             }),
     )
     .await?;
     // i just have a gut feeling this shouldn't be concurrent
-    for (dependency_id, info, dependency_manifest) in dependency_manifests {
+    for (dependency_id, info, dependency_manifest) in
+        dependency_manifests.into_iter().filter_map(|a| a)
+    {
         match (dependency_manifest.public, info.mount_public) {
             (Some(public), true) => {
                 let public_path = Path::new(crate::VOLUMES).join(&dependency_id).join(public);

appmgr/src/disks.rs

@@ -1,10 +1,11 @@
 use std::path::Path;
 
+use failure::ResultExt as _;
 use futures::future::try_join_all;
 
 use crate::util::Invoke;
 use crate::Error;
-use crate::ResultExt;
+use crate::ResultExt as _;
 
 pub const FSTAB: &'static str = "/etc/fstab";
 
@@ -153,6 +154,11 @@ pub async fn bind<P0: AsRef<Path>, P1: AsRef<Path>>(
     dst: P1,
     read_only: bool,
 ) -> Result<(), Error> {
+    log::info!(
+        "Binding {} to {}",
+        src.as_ref().display(),
+        dst.as_ref().display()
+    );
     let is_mountpoint = tokio::process::Command::new("mountpoint")
         .arg(dst.as_ref())
         .stdout(std::process::Stdio::null())
@@ -185,6 +191,7 @@ pub async fn bind<P0: AsRef<Path>, P1: AsRef<Path>>(
 }
 
 pub async fn unmount<P: AsRef<Path>>(mount_point: P) -> Result<(), Error> {
+    log::info!("Unmounting {}.", mount_point.as_ref().display());
     let umount_output = tokio::process::Command::new("umount")
         .arg(mount_point.as_ref())
         .output()
@@ -192,10 +199,14 @@ pub async fn unmount<P: AsRef<Path>>(mount_point: P) -> Result<(), Error> {
     crate::ensure_code!(
         umount_output.status.success(),
         crate::error::FILESYSTEM_ERROR,
-        "Error Unmounting Drive: {}",
+        "Error Unmounting Drive: {}: {}",
+        mount_point.as_ref().display(),
         std::str::from_utf8(&umount_output.stderr).unwrap_or("Unknown Error")
     );
-    tokio::fs::remove_dir_all(mount_point.as_ref()).await?;
+    tokio::fs::remove_dir_all(mount_point.as_ref())
+        .await
+        .with_context(|e| format!("rm {}: {}", mount_point.as_ref().display(), e))
+        .with_code(crate::error::FILESYSTEM_ERROR)?;
     Ok(())
 }
 

appmgr/src/index.rs

@@ -30,6 +30,7 @@ pub struct VersionInfo {
     pub release_notes: String,
     pub os_version_required: VersionRange,
     pub os_version_recommended: VersionRange,
+    pub install_alert: Option<String>,
 }
 
 const NULL_VERSION: Version = Version::new(0, 0, 0, 0);
@@ -52,6 +53,7 @@ impl AppIndex {
                 release_notes: manifest.release_notes,
                 os_version_required: manifest.os_version_required,
                 os_version_recommended: manifest.os_version_recommended,
+                install_alert: manifest.install_alert,
             });
             entry
                 .version_info
@@ -68,6 +70,7 @@ impl AppIndex {
                         release_notes: manifest.release_notes,
                         os_version_required: manifest.os_version_required,
                         os_version_recommended: manifest.os_version_recommended,
+                        install_alert: manifest.install_alert,
                     }],
                     icon_type: "png".to_owned(), // TODO
                 },

appmgr/src/install.rs

@@ -14,8 +14,9 @@ use std::time::Duration;
 use failure::ResultExt as _;
 use futures::stream::StreamExt;
 use futures::stream::TryStreamExt;
-use tokio::io::AsyncRead;
 use tokio::io::AsyncWriteExt;
+use tokio::io::{AsyncRead, ReadBuf};
+use tokio_compat_02::FutureExt;
 use tokio_tar as tar;
 
 use crate::config::{ConfigRuleEntry, ConfigSpec};
@@ -62,13 +63,13 @@ where
     fn poll_read(
         self: Pin<&mut Self>,
         cx: &mut Context<'_>,
-        buf: &mut [u8],
+        buf: &mut ReadBuf,
-    ) -> Poll<std::io::Result<usize>> {
+    ) -> Poll<std::io::Result<()>> {
         let atomic = self.as_ref().1.clone(); // TODO: not efficient
         match unsafe { self.map_unchecked_mut(|a| &mut a.0) }.poll_read(cx, buf) {
-            Poll::Ready(Ok(res)) => {
+            Poll::Ready(Ok(())) => {
-                atomic.fetch_add(res as u64, atomic::Ordering::SeqCst);
+                atomic.fetch_add(buf.filled().len() as u64, atomic::Ordering::SeqCst);
-                Poll::Ready(Ok(res))
+                Poll::Ready(Ok(()))
             }
             a => a,
         }
@@ -98,6 +99,7 @@ pub async fn download(url: &str, name: Option<&str>) -> Result<PathBuf, crate::E
     let url = reqwest::Url::parse(url).no_code()?;
     log::info!("Downloading {}.", url.as_str());
     let response = reqwest::get(url)
+        .compat()
         .await
         .with_code(crate::error::NETWORK_ERROR)?
         .error_for_status()
@@ -141,7 +143,7 @@ pub async fn download(url: &str, name: Option<&str>) -> Result<PathBuf, crate::E
             if is_done {
                 break;
             }
-            tokio::time::delay_for(Duration::from_millis(10)).await;
+            tokio::time::sleep(Duration::from_millis(10)).await;
         }
         if !*crate::QUIET.read().await {
             println!("\rDownloading... 100%");
@@ -191,7 +193,7 @@ pub async fn install_path<P: AsRef<Path>>(p: P, name: Option<&str>) -> Result<()
             if is_done {
                 break;
             }
-            tokio::time::delay_for(Duration::from_millis(10)).await;
+            tokio::time::sleep(Duration::from_millis(10)).await;
         }
         if !*crate::QUIET.read().await {
             println!("\rInstalling... 100%");
@@ -254,6 +256,19 @@ pub async fn install_v0<R: AsyncRead + Unpin + Send + Sync>(
             "Package Name Does Not Match Expected"
         );
     }
+
+    log::info!(
+        "Creating metadata directory: {}/apps/{}",
+        crate::PERSISTENCE_DIR,
+        manifest.id
+    );
+    let app_dir = PersistencePath::from_ref("apps").join(&manifest.id);
+    let app_dir_path = app_dir.path();
+    if app_dir_path.exists() {
+        tokio::fs::remove_dir_all(&app_dir_path).await?;
+    }
+    tokio::fs::create_dir_all(&app_dir_path).await?;
+
     let (ip, tor_addr, tor_key) = crate::tor::set_svc(
         &manifest.id,
         crate::tor::NewService {
@@ -268,12 +283,6 @@ pub async fn install_v0<R: AsyncRead + Unpin + Send + Sync>(
     log::info!("Creating volume {}/{}.", crate::VOLUMES, manifest.id);
     tokio::fs::create_dir_all(Path::new(crate::VOLUMES).join(&manifest.id)).await?;
 
-    let app_dir = PersistencePath::from_ref("apps").join(&manifest.id);
-    let app_dir_path = app_dir.path();
-    if app_dir_path.exists() {
-        tokio::fs::remove_dir_all(&app_dir_path).await?;
-    }
-    tokio::fs::create_dir_all(&app_dir_path).await?;
     let _lock = app_dir.lock(true).await?;
     log::info!("Saving manifest.");
     let mut manifest_out = app_dir.join("manifest.yaml").write(None).await?;
@@ -407,11 +416,13 @@ pub async fn install_v0<R: AsyncRead + Unpin + Send + Sync>(
                     .arg("stop")
                     .arg(&manifest.id)
                     .spawn()?
+                    .wait()
                     .await?;
                 tokio::process::Command::new("docker")
                     .arg("rm")
                     .arg(&manifest.id)
                     .spawn()?
+                    .wait()
                     .await?;
                 crate::ensure_code!(
                     tokio::process::Command::new("docker")
@@ -457,7 +468,7 @@ pub async fn install_v0<R: AsyncRead + Unpin + Send + Sync>(
             child_in.shutdown().await?;
             drop(child_in);
             crate::ensure_code!(
-                child.await?.success(),
+                child.wait().await?.success(),
                 crate::error::DOCKER_ERROR,
                 "Failed to Load Docker Image From Tar"
             );
@@ -474,7 +485,7 @@ pub async fn install_v0<R: AsyncRead + Unpin + Send + Sync>(
     let mut args = vec![
         Cow::Borrowed(OsStr::new("create")),
         Cow::Borrowed(OsStr::new("--restart")),
-        Cow::Borrowed(OsStr::new("on-failure")),
+        Cow::Borrowed(OsStr::new("no")),
         Cow::Borrowed(OsStr::new("--name")),
         Cow::Borrowed(OsStr::new(&manifest.id)),
         Cow::Borrowed(OsStr::new("--mount")),
@@ -550,13 +561,17 @@ pub async fn install_v0<R: AsyncRead + Unpin + Send + Sync>(
             crate::config::configure(&manifest.id, Some(empty_config), None, false).await?;
         }
     }
+    crate::dependencies::update_binds(&manifest.id).await?;
     for (dep_id, dep_info) in manifest.dependencies.0 {
         if dep_info.mount_shared
             && crate::apps::list_info().await?.get(&dep_id).is_some()
             && crate::apps::manifest(&dep_id).await?.shared.is_some()
-            && crate::apps::status(&dep_id).await?.status != crate::apps::DockerStatus::Stopped
         {
-            crate::apps::set_needs_restart(&dep_id, true).await?;
+            match crate::apps::status(&dep_id, false).await?.status {
+                crate::apps::DockerStatus::Stopped => (),
+                crate::apps::DockerStatus::Running => crate::control::restart_app(&dep_id).await?,
+                _ => crate::apps::set_needs_restart(&dep_id, true).await?,
+            }
         }
     }
 

appmgr/src/lan.rs

@@ -0,0 +1,93 @@
+use crate::Error;
+use avahi_sys;
+use futures::future::pending;
+
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+pub struct AppId {
+    pub un_app_id: String,
+}
+
+pub async fn enable_lan() -> Result<(), Error> {
+    unsafe {
+        let app_list = crate::apps::list_info().await?;
+
+        let simple_poll = avahi_sys::avahi_simple_poll_new();
+        let poll = avahi_sys::avahi_simple_poll_get(simple_poll);
+        let mut stack_err = 0;
+        let err_c: *mut i32 = &mut stack_err;
+        let avahi_client = avahi_sys::avahi_client_new(
+            poll,
+            avahi_sys::AvahiClientFlags::AVAHI_CLIENT_NO_FAIL,
+            None,
+            std::ptr::null_mut(),
+            err_c,
+        );
+        let group =
+            avahi_sys::avahi_entry_group_new(avahi_client, Some(noop), std::ptr::null_mut());
+        let hostname_raw = avahi_sys::avahi_client_get_host_name_fqdn(avahi_client);
+        let hostname_bytes = std::ffi::CStr::from_ptr(hostname_raw).to_bytes_with_nul();
+        const HOSTNAME_LEN: usize = 1 + 15 + 1 + 5; // leading byte, main address, dot, "local"
+        debug_assert_eq!(hostname_bytes.len(), HOSTNAME_LEN);
+        let mut hostname_buf = [0; HOSTNAME_LEN + 1];
+        hostname_buf[1..].copy_from_slice(hostname_bytes);
+        // assume fixed length prefix on hostname due to local address
+        hostname_buf[0] = 15; // set the prefix length to 15 for the main address
+        hostname_buf[16] = 5; // set the prefix length to 5 for "local"
+
+        for (app_id, app_info) in app_list {
+            let man = crate::apps::manifest(&app_id).await?;
+            if man
+                .ports
+                .iter()
+                .filter(|p| p.lan.is_some())
+                .next()
+                .is_none()
+            {
+                continue;
+            }
+            let tor_address = if let Some(addr) = app_info.tor_address {
+                addr
+            } else {
+                continue;
+            };
+            let lan_address = tor_address
+                .strip_suffix(".onion")
+                .ok_or_else(|| failure::format_err!("Invalid Tor Address: {:?}", tor_address))?
+                .to_owned()
+                + ".local";
+            let lan_address_ptr = std::ffi::CString::new(lan_address)
+                .expect("Could not cast lan address to c string");
+            let _ = avahi_sys::avahi_entry_group_add_record(
+                group,
+                avahi_sys::AVAHI_IF_UNSPEC,
+                avahi_sys::AVAHI_PROTO_UNSPEC,
+                avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_USE_MULTICAST
+                    | avahi_sys::AvahiPublishFlags_AVAHI_PUBLISH_ALLOW_MULTIPLE,
+                lan_address_ptr.as_ptr(),
+                avahi_sys::AVAHI_DNS_CLASS_IN as u16,
+                avahi_sys::AVAHI_DNS_TYPE_CNAME as u16,
+                avahi_sys::AVAHI_DEFAULT_TTL,
+                hostname_buf.as_ptr().cast(),
+                hostname_buf.len(),
+            );
+            log::info!("Published {:?}", lan_address_ptr);
+        }
+        avahi_sys::avahi_entry_group_commit(group);
+        ctrlc::set_handler(move || {
+            // please the borrow checker with the below semantics
+            // avahi_sys::avahi_entry_group_free(group);
+            // avahi_sys::avahi_client_free(avahi_client);
+            // drop(Box::from_raw(err_c));
+            std::process::exit(0);
+        })
+        .expect("Error setting signal handler");
+    }
+    pending().await
+}
+
+unsafe extern "C" fn noop(
+    _group: *mut avahi_sys::AvahiEntryGroup,
+    _state: avahi_sys::AvahiEntryGroupState,
+    _userdata: *mut core::ffi::c_void,
+) {
+}

appmgr/src/lib.rs

@@ -20,6 +20,7 @@ lazy_static::lazy_static! {
     pub static ref QUIET: tokio::sync::RwLock<bool> = tokio::sync::RwLock::new(!std::env::var("APPMGR_QUIET").map(|a| a == "0").unwrap_or(true));
 }
 
+pub mod actions;
 pub mod apps;
 pub mod backup;
 pub mod config;
@@ -30,6 +31,8 @@ pub mod error;
 pub mod index;
 pub mod inspect;
 pub mod install;
+#[cfg(feature = "avahi")]
+pub mod lan;
 pub mod logs;
 pub mod manifest;
 pub mod pack;

appmgr/src/main.rs

@@ -162,6 +162,17 @@ async fn inner_main() -> Result<(), Error> {
                 ),
         );
 
+    #[cfg(feature = "avahi")]
+    #[allow(unused_mut)]
+    let mut app = app.subcommand(
+        SubCommand::with_name("lan")
+            .about("Configures LAN services")
+            .subcommand(
+                SubCommand::with_name("enable")
+                    .about("Publishes the LAN addresses for all services"),
+            ),
+    );
+
     #[cfg(not(feature = "portable"))]
     let mut app = app
         .subcommand(
@@ -399,7 +410,6 @@ async fn inner_main() -> Result<(), Error> {
                 .about("Removes an installed app")
                 .arg(
                     Arg::with_name("purge")
-                        .short("p")
                         .long("purge")
                         .help("Deletes all application data"),
                 )
@@ -817,6 +827,19 @@ async fn inner_main() -> Result<(), Error> {
                                 .help("Password to use for encryption of backup file"),
                         ),
                 ),
+        )
+        .subcommand(
+            SubCommand::with_name("repair-app-status").about("Restarts crashed apps"), // TODO: remove
+        )
+        .subcommand(
+            SubCommand::with_name("actions")
+                .about("Perform an action for a service")
+                .arg(
+                    Arg::with_name("SERVICE")
+                        .help("ID of the service to perform an action on")
+                        .required(true),
+                )
+                .arg(Arg::with_name("ACTION").help("ID of the action to perform")),
         );
 
     let matches = app.clone().get_matches();
@@ -1175,6 +1198,15 @@ async fn inner_main() -> Result<(), Error> {
                 std::process::exit(1);
             }
         },
+        #[cfg(feature = "avahi")]
+        #[cfg(not(feature = "portable"))]
+        ("lan", Some(sub_m)) => match sub_m.subcommand() {
+            ("enable", _) => crate::lan::enable_lan().await?,
+            _ => {
+                println!("{}", sub_m.usage());
+                std::process::exit(1);
+            }
+        },
         #[cfg(not(feature = "portable"))]
         ("info", Some(sub_m)) => {
             let name = sub_m.value_of("ID").unwrap();
@@ -1540,6 +1572,38 @@ async fn inner_main() -> Result<(), Error> {
                 std::process::exit(1);
             }
         },
+        #[cfg(not(feature = "portable"))]
+        ("repair-app-status", _) => {
+            control::repair_app_status().await?;
+        }
+        #[cfg(not(feature = "portable"))]
+        ("actions", Some(sub_m)) => {
+            use yajrc::{GenericRpcMethod, RpcResponse};
+
+            let man = apps::manifest(sub_m.value_of("SERVICE").unwrap()).await?;
+            let action_id = sub_m.value_of("ACTION").unwrap();
+            println!(
+                "{}",
+                serde_json::to_string(&RpcResponse::<GenericRpcMethod>::from_result(
+                    man.actions
+                        .iter()
+                        .filter(|a| &a.id == &action_id)
+                        .next()
+                        .ok_or_else(|| {
+                            failure::format_err!(
+                                "action {} does not exist for {}",
+                                action_id,
+                                man.id
+                            )
+                        })
+                        .with_code(error::NOT_FOUND)?
+                        .perform(&man.id)
+                        .await
+                        .map(serde_json::Value::String)
+                ))
+                .with_code(error::SERDE_ERROR)?
+            )
+        }
         ("pack", Some(sub_m)) => {
             pack(
                 sub_m.value_of("PATH").unwrap(),

appmgr/src/manifest.rs

@@ -2,6 +2,7 @@ use std::path::PathBuf;
 
 use linear_map::LinearMap;
 
+use crate::actions::Action;
 use crate::dependencies::Dependencies;
 use crate::tor::HiddenServiceVersion;
 use crate::tor::PortMapping;
@@ -37,6 +38,14 @@ pub struct ManifestV0 {
     pub description: Description,
     pub release_notes: String,
     #[serde(default)]
+    pub install_alert: Option<String>,
+    #[serde(default)]
+    pub uninstall_alert: Option<String>,
+    #[serde(default)]
+    pub restore_alert: Option<String>,
+    #[serde(default)]
+    pub start_alert: Option<String>,
+    #[serde(default)]
     pub has_instructions: bool,
     #[serde(default = "emver::VersionRange::any")]
     pub os_version_required: emver::VersionRange,
@@ -57,6 +66,8 @@ pub struct ManifestV0 {
     pub hidden_service_version: HiddenServiceVersion,
     #[serde(default)]
     pub dependencies: Dependencies,
+    #[serde(default)]
+    pub actions: Vec<Action>,
     #[serde(flatten)]
     pub extra: LinearMap<String, serde_yaml::Value>,
 }

appmgr/src/nginx-standard.conf.template

@@ -0,0 +1,28 @@
+map $http_x_forwarded_proto $real_proto {{
+    ext+onions  ext+onions;
+    ext+onion   ext+onion;
+    https       https;
+    http        http;
+    default     $scheme;
+}}
+server {{
+    listen 443 ssl;
+    server_name {hostname}.local;
+    ssl_certificate /root/appmgr/apps/{app_id}/cert-local.fullchain.crt.pem;
+    ssl_certificate_key /root/appmgr/apps/{app_id}/cert-local.key.pem;
+    location / {{
+        proxy_pass http://{app_ip}:{internal_port}/;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $real_proto;
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering off;
+    }}
+}}
+server {{
+    listen 80;
+    server_name {hostname}.local;
+    return 301 https://$host$request_uri;
+}}

appmgr/src/nginx.conf.template

@@ -0,0 +1,11 @@
+server {{
+    listen {port};
+    server_name {hostname}.local;
+    location / {{
+        proxy_pass http://{app_ip}:{internal_port}/;
+        proxy_set_header Host $host;
+        client_max_body_size 0;
+        proxy_request_buffering off;
+        proxy_buffering off;
+    }}
+}}

appmgr/src/pack.rs

@@ -25,14 +25,6 @@ pub enum Error {
 pub async fn pack(path: &str, output: &str) -> Result<(), failure::Error> {
     let path = Path::new(path.trim_end_matches("/"));
     let output = Path::new(output);
-    ensure!(
-        output
-            .extension()
-            .and_then(|a| a.to_str())
-            .ok_or_else(|| Error::InvalidOutputPath(format!("{}", output.display())))?
-            == "s9pk",
-        "Extension Must Be '.s9pk'"
-    );
     log::info!(
         "Starting pack of {} to {}.",
         path.file_name()
@@ -74,9 +66,6 @@ pub async fn pack(path: &str, output: &str) -> Result<(), failure::Error> {
             .with_context(|e| format!("{}: config_spec.yaml", e))?,
     )
     .await?;
-    config_spec.validate(&manifest)?;
-    let config = config_spec.gen(&mut rand::rngs::StdRng::from_entropy(), &None)?;
-    config_spec.matches(&config)?;
     log::info!("Writing config spec to archive.");
     let bin_config_spec = serde_cbor::to_vec(&config_spec)?;
     let mut config_spec_header = tar::Header::new_gnu();
@@ -94,12 +83,6 @@ pub async fn pack(path: &str, output: &str) -> Result<(), failure::Error> {
             .with_context(|e| format!("{}: config_rules.yaml", e))?,
     )
     .await?;
-    let mut cfgs = LinearMap::new();
-    cfgs.insert(manifest.id.as_str(), Cow::Borrowed(&config));
-    for rule in &config_rules {
-        rule.check(&config, &cfgs)
-            .with_context(|e| format!("Default Config does not satisfy: {}", e))?;
-    }
     log::info!("Writing config rules to archive.");
     let bin_config_rules = serde_cbor::to_vec(&config_rules)?;
     let mut config_rules_header = tar::Header::new_gnu();
@@ -234,6 +217,13 @@ pub async fn verify(path: &str) -> Result<(), failure::Error> {
     if let Some(shared) = &manifest.shared {
         validate_path(shared)?;
     }
+    for action in &manifest.actions {
+        ensure!(
+            !action.command.is_empty(),
+            "Command Cannot Be Empty: {}",
+            action.id
+        );
+    }
     log::info!("Opening config spec from archive.");
     let config_spec = entries
         .next()

appmgr/src/registry.rs

@@ -1,4 +1,5 @@
 use emver::VersionRange;
+use tokio_compat_02::FutureExt;
 
 use crate::apps::AppConfig;
 use crate::manifest::ManifestLatest;
@@ -12,6 +13,7 @@ pub async fn manifest(id: &str, version: &VersionRange) -> Result<ManifestLatest
         id,
         version
     ))
+    .compat()
     .await
     .with_code(crate::error::NETWORK_ERROR)?
     .error_for_status()
@@ -34,6 +36,7 @@ pub async fn version(id: &str, version: &VersionRange) -> Result<emver::Version,
         id,
         version
     ))
+    .compat()
     .await
     .with_code(crate::error::NETWORK_ERROR)?
     .error_for_status()
@@ -51,6 +54,7 @@ pub async fn config(id: &str, version: &VersionRange) -> Result<AppConfig, Error
         id,
         version
     ))
+    .compat()
     .await
     .with_code(crate::error::NETWORK_ERROR)?
     .error_for_status()

appmgr/src/remove.rs

@@ -1,9 +1,11 @@
+use crate::failure::ResultExt;
 use std::path::Path;
 
 use linear_map::LinearMap;
 
 use crate::dependencies::{DependencyError, TaggedDependencyError};
 use crate::Error;
+use crate::ResultExt as _;
 
 pub async fn remove(
     name: &str,
@@ -55,48 +57,79 @@ pub async fn remove(
         log::info!("Removing tor hidden service.");
         crate::tor::rm_svc(name).await?;
         log::info!("Removing app metadata.");
-        tokio::fs::remove_dir_all(Path::new(crate::PERSISTENCE_DIR).join("apps").join(name))
+        let metadata_path = Path::new(crate::PERSISTENCE_DIR).join("apps").join(name);
-            .await?;
+        tokio::fs::remove_dir_all(&metadata_path)
-        log::info!("Destroying mounted volume.");
+            .await
+            .with_context(|e| format!("rm {}: {}", metadata_path.display(), e))
+            .with_code(crate::error::FILESYSTEM_ERROR)?;
         log::info!("Unbinding shared filesystem.");
-        for (dep, info) in manifest.dependencies.0.iter() {
+        let installed_apps = crate::apps::list_info().await?;
-            if info.mount_public {
+        for (dep, _) in manifest.dependencies.0.iter() {
-                crate::disks::unmount(
+            let path = Path::new(crate::VOLUMES)
-                    Path::new(crate::VOLUMES)
+                .join(name)
-                        .join(name)
+                .join("start9")
-                        .join("start9")
+                .join("public")
-                        .join("public")
+                .join(&dep);
-                        .join(&dep),
+            if path.exists() {
-                )
+                crate::disks::unmount(&path).await?;
-                .await?;
+            } else {
+                log::warn!("{} does not exist, skipping...", path.display());
             }
-            if info.mount_shared {
+            let path = Path::new(crate::VOLUMES)
-                if let Some(shared) = match crate::apps::manifest(dep).await {
+                .join(name)
-                    Ok(man) => man.shared,
+                .join("start9")
-                    Err(e) => {
+                .join("shared")
-                        log::error!("Failed to Fetch Dependency Manifest: {}", e);
+                .join(&dep);
-                        None
+            if path.exists() {
-                    }
+                crate::disks::unmount(&path).await?;
-                } {
+            } else {
-                    let path = Path::new(crate::VOLUMES)
+                log::warn!("{} does not exist, skipping...", path.display());
-                        .join(name)
+            }
-                        .join("start9")
+            if installed_apps.contains_key(dep) {
-                        .join("shared")
+                let dep_man = crate::apps::manifest(dep).await?;
-                        .join(&dep);
+                if let Some(shared) = dep_man.shared {
-                    if path.exists() {
-                        crate::disks::unmount(&path).await?;
-                    }
                     let path = Path::new(crate::VOLUMES).join(dep).join(&shared).join(name);
                     if path.exists() {
-                        tokio::fs::remove_dir_all(
+                        tokio::fs::remove_dir_all(&path)
-                            Path::new(crate::VOLUMES).join(dep).join(&shared).join(name),
+                            .await
-                        )
+                            .with_context(|e| format!("rm {}: {}", path.display(), e))
-                        .await?;
+                            .with_code(crate::error::FILESYSTEM_ERROR)?;
                     }
                 }
+            } else {
+                log::warn!("{} is not installed, skipping...", dep);
+            }
+        }
+        if manifest.public.is_some() || manifest.shared.is_some() {
+            for dependent in crate::apps::dependents(name, false).await? {
+                let path = Path::new(crate::VOLUMES)
+                    .join(&dependent)
+                    .join("start9")
+                    .join("public")
+                    .join(name);
+                if path.exists() {
+                    crate::disks::unmount(&path).await?;
+                } else {
+                    log::warn!("{} does not exist, skipping...", path.display());
+                }
+                let path = Path::new(crate::VOLUMES)
+                    .join(dependent)
+                    .join("start9")
+                    .join("shared")
+                    .join(name);
+                if path.exists() {
+                    crate::disks::unmount(&path).await?;
+                } else {
+                    log::warn!("{} does not exist, skipping...", path.display());
+                }
             }
         }
-        tokio::fs::remove_dir_all(Path::new(crate::VOLUMES).join(name)).await?;
+        log::info!("Destroying mounted volume.");
+        let volume_path = Path::new(crate::VOLUMES).join(name);
+        tokio::fs::remove_dir_all(&volume_path)
+            .await
+            .with_context(|e| format!("rm {}: {}", volume_path.display(), e))
+            .with_code(crate::error::FILESYSTEM_ERROR)?;
         log::info!("Pruning unused docker images.");
         crate::ensure_code!(
             std::process::Command::new("docker")

appmgr/src/tor.rs

@@ -8,17 +8,62 @@ use failure::ResultExt as _;
 use tokio::io::AsyncReadExt;
 use tokio::io::AsyncWriteExt;
 
-use crate::util::{PersistencePath, YamlUpdateHandle};
+use crate::util::{Invoke, PersistencePath, YamlUpdateHandle};
 use crate::{Error, ResultExt as _};
 
 #[derive(Debug, Clone, Copy, serde::Deserialize, serde::Serialize)]
+#[serde(rename_all = "kebab-case")]
+pub enum LanOptions {
+    Standard,
+    Custom { port: u16 },
+}
+
+#[derive(Debug, Clone, Copy, serde::Serialize)]
 pub struct PortMapping {
     pub internal: u16,
     pub tor: u16,
+    pub lan: Option<LanOptions>, // only for http interfaces
+}
+impl<'de> serde::de::Deserialize<'de> for PortMapping {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: serde::de::Deserializer<'de>,
+    {
+        #[derive(serde::Deserialize)]
+        pub struct PortMappingIF {
+            pub internal: u16,
+            pub tor: u16,
+            #[serde(default, deserialize_with = "deserialize_some")]
+            pub lan: Option<Option<LanOptions>>,
+        }
+
+        fn deserialize_some<'de, T, D>(deserializer: D) -> Result<Option<T>, D::Error>
+        where
+            T: serde::de::Deserialize<'de>,
+            D: serde::de::Deserializer<'de>,
+        {
+            serde::de::Deserialize::deserialize(deserializer).map(Some)
+        }
+
+        let input_format: PortMappingIF = serde::de::Deserialize::deserialize(deserializer)?;
+        Ok(PortMapping {
+            internal: input_format.internal,
+            tor: input_format.tor,
+            lan: if let Some(lan) = input_format.lan {
+                lan
+            } else if input_format.tor == 80 {
+                Some(LanOptions::Standard)
+            } else {
+                None
+            },
+        })
+    }
 }
 
 pub const ETC_TOR_RC: &'static str = "/etc/tor/torrc";
 pub const HIDDEN_SERVICE_DIR_ROOT: &'static str = "/var/lib/tor";
+pub const ETC_HOSTNAME: &'static str = "/etc/hostname";
+pub const ETC_NGINX_SERVICES_CONF: &'static str = "/etc/nginx/sites-available/start9-services.conf";
 
 #[derive(Debug, Clone, Copy, serde::Deserialize, serde::Serialize)]
 #[serde(rename_all = "lowercase")]
@@ -179,6 +224,161 @@ pub async fn write_services(hidden_services: &ServicesMap) -> Result<(), Error>
     Ok(())
 }
 
+pub async fn write_lan_services(hidden_services: &ServicesMap) -> Result<(), Error> {
+    let mut f = tokio::fs::File::create(ETC_NGINX_SERVICES_CONF).await?;
+    for (app_id, service) in &hidden_services.map {
+        let hostname = tokio::fs::read_to_string(
+            Path::new(HIDDEN_SERVICE_DIR_ROOT)
+                .join(format!("app-{}", app_id))
+                .join("hostname"),
+        )
+        .await
+        .with_context(|e| format!("{}/app-{}/hostname: {}", HIDDEN_SERVICE_DIR_ROOT, app_id, e))
+        .with_code(crate::error::FILESYSTEM_ERROR)?;
+        let hostname_str = hostname
+            .trim()
+            .strip_suffix(".onion")
+            .ok_or_else(|| failure::format_err!("invalid tor hostname"))
+            .no_code()?;
+        for mapping in &service.ports {
+            match &mapping.lan {
+                Some(LanOptions::Standard) => {
+                    log::info!("Writing LAN certificates for {}", app_id);
+                    let base_path = PersistencePath::from_ref("apps").join(&app_id);
+                    let key_path = base_path.join("cert-local.key.pem").path();
+                    let conf_path = base_path.join("cert-local.csr.conf").path();
+                    let req_path = base_path.join("cert-local.csr").path();
+                    let cert_path = base_path.join("cert-local.crt.pem").path();
+                    let fullchain_path = base_path.join("cert-local.fullchain.crt.pem");
+                    if !fullchain_path.exists().await
+                        || tokio::fs::metadata(&key_path).await.is_err()
+                    {
+                        let mut fullchain_file = fullchain_path.write(None).await?;
+                        tokio::process::Command::new("openssl")
+                            .arg("ecparam")
+                            .arg("-genkey")
+                            .arg("-name")
+                            .arg("prime256v1")
+                            .arg("-noout")
+                            .arg("-out")
+                            .arg(&key_path)
+                            .invoke("OpenSSL GenKey")
+                            .await?;
+                        tokio::fs::write(
+                            &conf_path,
+                            format!(
+                                include_str!("cert-local.csr.conf.template"),
+                                hostname = hostname_str
+                            ),
+                        )
+                        .await?;
+                        tokio::process::Command::new("openssl")
+                            .arg("req")
+                            .arg("-config")
+                            .arg(&conf_path)
+                            .arg("-key")
+                            .arg(&key_path)
+                            .arg("-new")
+                            .arg("-addext")
+                            .arg(format!(
+                                "subjectAltName=DNS:{hostname}.local",
+                                hostname = hostname_str
+                            ))
+                            .arg("-out")
+                            .arg(&req_path)
+                            .invoke("OpenSSL Req")
+                            .await?;
+                        tokio::process::Command::new("openssl")
+                            .arg("ca")
+                            .arg("-batch")
+                            .arg("-config")
+                            .arg("/root/agent/ca/intermediate/openssl.conf")
+                            .arg("-rand_serial")
+                            .arg("-keyfile")
+                            .arg("/root/agent/ca/intermediate/private/embassy-int-ca.key.pem")
+                            .arg("-cert")
+                            .arg("/root/agent/ca/intermediate/certs/embassy-int-ca.crt.pem")
+                            .arg("-extensions")
+                            .arg("server_cert")
+                            .arg("-days")
+                            .arg("365")
+                            .arg("-notext")
+                            .arg("-in")
+                            .arg(&req_path)
+                            .arg("-out")
+                            .arg(&cert_path)
+                            .invoke("OpenSSL CA")
+                            .await?;
+                        log::info!("Writing fullchain to: {}", fullchain_path.path().display());
+                        tokio::io::copy(
+                            &mut tokio::fs::File::open(&cert_path).await?,
+                            &mut *fullchain_file,
+                        )
+                        .await?;
+                        tokio::io::copy(
+                            &mut tokio::fs::File::open(
+                                "/root/agent/ca/intermediate/certs/embassy-int-ca.crt.pem",
+                            )
+                            .await
+                            .with_context(|e| {
+                                format!(
+                                    "{}: /root/agent/ca/intermediate/certs/embassy-int-ca.crt.pem",
+                                    e
+                                )
+                            })
+                            .with_code(crate::error::FILESYSTEM_ERROR)?,
+                            &mut *fullchain_file,
+                        )
+                        .await?;
+                        tokio::io::copy(
+                            &mut tokio::fs::File::open(
+                                "/root/agent/ca/certs/embassy-root-ca.cert.pem",
+                            )
+                            .await
+                            .with_context(|e| {
+                                format!("{}: /root/agent/ca/certs/embassy-root-ca.cert.pem", e)
+                            })
+                            .with_code(crate::error::FILESYSTEM_ERROR)?,
+                            &mut *fullchain_file,
+                        )
+                        .await?;
+                        fullchain_file.commit().await?;
+                        log::info!("{} written successfully", fullchain_path.path().display());
+                    }
+                    f.write_all(
+                        format!(
+                            include_str!("nginx-standard.conf.template"),
+                            hostname = hostname_str,
+                            app_ip = service.ip,
+                            internal_port = mapping.internal,
+                            app_id = app_id,
+                        )
+                        .as_bytes(),
+                    )
+                    .await?;
+                    f.sync_all().await?;
+                }
+                Some(LanOptions::Custom { port }) => {
+                    f.write_all(
+                        format!(
+                            include_str!("nginx.conf.template"),
+                            hostname = hostname_str,
+                            app_ip = service.ip,
+                            port = port,
+                            internal_port = mapping.internal,
+                        )
+                        .as_bytes(),
+                    )
+                    .await?
+                }
+                None => (),
+            }
+        }
+    }
+
+    Ok(())
+}
+
 pub async fn read_tor_address(name: &str, timeout: Option<Duration>) -> Result<String, Error> {
     log::info!("Retrieving Tor hidden service address for {}.", name);
     let addr_path = Path::new(HIDDEN_SERVICE_DIR_ROOT)
@@ -198,7 +398,7 @@ pub async fn read_tor_address(name: &str, timeout: Option<Duration>) -> Result<S
                 }
             }
         } {
-            tokio::time::delay_for(Duration::from_millis(100)).await;
+            tokio::time::sleep(Duration::from_millis(100)).await;
         }
     }
     let tor_addr = match tokio::fs::read_to_string(&addr_path).await {
@@ -217,7 +417,7 @@ pub async fn read_tor_key(
     version: HiddenServiceVersion,
     timeout: Option<Duration>,
 ) -> Result<String, Error> {
-    log::info!("Retrieving Tor hidden service address for {}.", name);
+    log::info!("Retrieving Tor hidden service key for {}.", name);
     let addr_path = Path::new(HIDDEN_SERVICE_DIR_ROOT)
         .join(format!("app-{}", name))
         .join(match version {
@@ -238,7 +438,7 @@ pub async fn read_tor_key(
                 }
             }
         } {
-            tokio::time::delay_for(Duration::from_millis(100)).await;
+            tokio::time::sleep(Duration::from_millis(100)).await;
         }
     }
     let tor_key = match version {
@@ -277,7 +477,18 @@ pub async fn set_svc(
     let ip = hidden_services.add(name.to_owned(), service);
     log::info!("Adding Tor hidden service {} to {}.", name, ETC_TOR_RC);
     write_services(&hidden_services).await?;
-    hidden_services.commit().await?;
+    let addr_path = Path::new(HIDDEN_SERVICE_DIR_ROOT)
+        .join(format!("app-{}", name))
+        .join("hostname");
+    tokio::fs::remove_file(addr_path).await.or_else(|e| {
+        if e.kind() == std::io::ErrorKind::NotFound {
+            Ok(())
+        } else {
+            Err(e)
+        }
+    })?;
+    #[cfg(target_os = "linux")]
+    nix::unistd::sync();
     log::info!("Reloading Tor.");
     let svc_exit = std::process::Command::new("service")
         .args(&["tor", "reload"])
@@ -291,19 +502,32 @@ pub async fn set_svc(
             .or_else(|| { svc_exit.signal().map(|a| 128 + a) })
             .unwrap_or(0)
     );
-    Ok((
+    let addr = if is_listening {
-        ip,
+        Some(read_tor_address(name, Some(Duration::from_secs(30))).await?)
-        if is_listening {
+    } else {
-            Some(read_tor_address(name, Some(Duration::from_secs(30))).await?)
+        None
-        } else {
+    };
-            None
+    let key = if is_listening {
-        },
+        Some(read_tor_key(name, ver, Some(Duration::from_secs(30))).await?)
-        if is_listening {
+    } else {
-            Some(read_tor_key(name, ver, Some(Duration::from_secs(30))).await?)
+        None
-        } else {
+    };
-            None
+    write_lan_services(&hidden_services).await?;
-        },
+    log::info!("Reloading Nginx.");
-    ))
+    let svc_exit = std::process::Command::new("service")
+        .args(&["nginx", "reload"])
+        .status()?;
+    crate::ensure_code!(
+        svc_exit.success(),
+        crate::error::GENERAL_ERROR,
+        "Failed to Reload Nginx: {}",
+        svc_exit
+            .code()
+            .or_else(|| { svc_exit.signal().map(|a| 128 + a) })
+            .unwrap_or(0)
+    );
+    hidden_services.commit().await?;
+    Ok((ip, addr, key))
 }
 
 pub async fn rm_svc(name: &str) -> Result<(), Error> {
@@ -322,7 +546,6 @@ pub async fn rm_svc(name: &str) -> Result<(), Error> {
     }
     log::info!("Removing Tor hidden service {} from {}.", name, ETC_TOR_RC);
     write_services(&hidden_services).await?;
-    hidden_services.commit().await?;
     log::info!("Reloading Tor.");
     let svc_exit = std::process::Command::new("service")
         .args(&["tor", "reload"])
@@ -333,6 +556,21 @@ pub async fn rm_svc(name: &str) -> Result<(), Error> {
         "Failed to Reload Tor: {}",
         svc_exit.code().unwrap_or(0)
     );
+    write_lan_services(&hidden_services).await?;
+    log::info!("Reloading Nginx.");
+    let svc_exit = std::process::Command::new("service")
+        .args(&["nginx", "reload"])
+        .status()?;
+    crate::ensure_code!(
+        svc_exit.success(),
+        crate::error::GENERAL_ERROR,
+        "Failed to Reload Nginx: {}",
+        svc_exit
+            .code()
+            .or_else(|| { svc_exit.signal().map(|a| 128 + a) })
+            .unwrap_or(0)
+    );
+    hidden_services.commit().await?;
     Ok(())
 }
 

appmgr/src/update.rs

@@ -19,7 +19,9 @@ pub async fn update(
     let version = crate::registry::version(name, &version_req).await?;
     let mut res = LinearMap::new();
     for dependent in crate::apps::dependents(name, false).await? {
-        if crate::apps::status(&dependent).await?.status != crate::apps::DockerStatus::Stopped {
+        if crate::apps::status(&dependent, false).await?.status
+            != crate::apps::DockerStatus::Stopped
+        {
             let manifest = crate::apps::manifest(&dependent).await?;
             match manifest.dependencies.0.get(name) {
                 Some(dep) if !version.satisfies(&dep.version) => {
@@ -30,7 +32,8 @@ pub async fn update(
                         &mut res,
                     )
                     .await?;
-                    if crate::apps::status(name).await?.status != crate::apps::DockerStatus::Stopped
+                    if crate::apps::status(name, false).await?.status
+                        != crate::apps::DockerStatus::Stopped
                     {
                         crate::control::stop_app(&dependent, false, dry_run).await?;
                         res.insert(
@@ -53,7 +56,8 @@ pub async fn update(
                         &mut res,
                     )
                     .await?;
-                    if crate::apps::status(name).await?.status != crate::apps::DockerStatus::Stopped
+                    if crate::apps::status(name, false).await?.status
+                        != crate::apps::DockerStatus::Stopped
                     {
                         crate::control::stop_app(&dependent, false, dry_run).await?;
                         res.insert(

appmgr/src/util.rs

@@ -5,7 +5,7 @@ use std::path::{Path, PathBuf};
 use failure::ResultExt as _;
 use file_lock::FileLock;
 use tokio::fs::File;
-use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt, ReadBuf};
 
 use crate::Error;
 use crate::ResultExt as _;
@@ -110,6 +110,14 @@ impl PersistencePath {
     pub async fn for_update(self) -> Result<UpdateHandle<ForRead>, Error> {
         UpdateHandle::new(self).await
     }
+
+    pub async fn delete(&self) -> Result<(), Error> {
+        match tokio::fs::remove_file(self.path()).await {
+            Ok(()) => Ok(()),
+            Err(k) if k.kind() == std::io::ErrorKind::NotFound => Ok(()),
+            e => e.with_code(crate::error::FILESYSTEM_ERROR),
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -137,6 +145,7 @@ impl PersistenceFile {
         if let Some(mut file) = self.file.take() {
             file.flush().await?;
             file.shutdown().await?;
+            file.sync_all().await?;
             drop(file);
         }
         if let Some(path) = self.needs_commit.take() {
@@ -156,10 +165,6 @@ impl PersistenceFile {
                     .await
                     .with_context(|e| format!("{}.lock: {}", path.path().display(), e))
                     .with_code(crate::error::FILESYSTEM_ERROR)?;
-                tokio::fs::remove_file(format!("{}.lock", path.path().display()))
-                    .await
-                    .with_context(|e| format!("{}.lock: {}", path.path().display(), e))
-                    .with_code(crate::error::FILESYSTEM_ERROR)?;
             }
 
             Ok(())
@@ -244,8 +249,8 @@ impl tokio::io::AsyncRead for UpdateHandle<ForRead> {
     fn poll_read(
         self: std::pin::Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
-        buf: &mut [u8],
+        buf: &mut ReadBuf,
-    ) -> std::task::Poll<std::io::Result<usize>> {
+    ) -> std::task::Poll<std::io::Result<()>> {
         unsafe { self.map_unchecked_mut(|a| a.file.file.as_mut().unwrap()) }.poll_read(cx, buf)
     }
 }
@@ -371,7 +376,13 @@ where
         cx: &mut std::task::Context<'_>,
         buf: &mut [u8],
     ) -> std::task::Poll<std::io::Result<usize>> {
-        tokio::io::AsyncRead::poll_read(unsafe { self.map_unchecked_mut(|a| &mut a.0) }, cx, buf)
+        let mut read_buf = ReadBuf::new(buf);
+        tokio::io::AsyncRead::poll_read(
+            unsafe { self.map_unchecked_mut(|a| &mut a.0) },
+            cx,
+            &mut read_buf,
+        )
+        .map(|res| res.map(|_| read_buf.filled().len()))
     }
 }
 impl<T> tokio::io::AsyncRead for AsyncCompat<T>
@@ -381,9 +392,14 @@ where
     fn poll_read(
         self: std::pin::Pin<&mut Self>,
         cx: &mut std::task::Context<'_>,
-        buf: &mut [u8],
+        buf: &mut ReadBuf,
-    ) -> std::task::Poll<std::io::Result<usize>> {
+    ) -> std::task::Poll<std::io::Result<()>> {
-        futures::io::AsyncRead::poll_read(unsafe { self.map_unchecked_mut(|a| &mut a.0) }, cx, buf)
+        futures::io::AsyncRead::poll_read(
+            unsafe { self.map_unchecked_mut(|a| &mut a.0) },
+            cx,
+            buf.initialize_unfilled(),
+        )
+        .map(|res| res.map(|len| buf.set_filled(len)))
     }
 }
 impl<T> futures::io::AsyncWrite for AsyncCompat<T>

appmgr/src/version/mod.rs

@@ -3,6 +3,7 @@ use std::cmp::Ordering;
 use async_trait::async_trait;
 use failure::ResultExt as _;
 use futures::stream::TryStreamExt;
+use tokio_compat_02::FutureExt;
 
 use crate::util::{to_yaml_async_writer, AsyncCompat, PersistencePath};
 use crate::Error;
@@ -21,8 +22,19 @@ mod v0_2_3;
 mod v0_2_4;
 mod v0_2_5;
 mod v0_2_6;
+mod v0_2_7;
+mod v0_2_8;
+mod v0_2_9;
 
-pub use v0_2_6::Version as Current;
+mod v0_2_10;
+mod v0_2_11;
+mod v0_2_12;
+mod v0_2_13;
+mod v0_2_14;
+mod v0_2_15;
+mod v0_2_16;
+
+pub use v0_2_16::Version as Current;
 
 #[derive(serde::Serialize, serde::Deserialize)]
 #[serde(untagged)]
@@ -41,6 +53,16 @@ enum Version {
     V0_2_4(Wrapper<v0_2_4::Version>),
     V0_2_5(Wrapper<v0_2_5::Version>),
     V0_2_6(Wrapper<v0_2_6::Version>),
+    V0_2_7(Wrapper<v0_2_7::Version>),
+    V0_2_8(Wrapper<v0_2_8::Version>),
+    V0_2_9(Wrapper<v0_2_9::Version>),
+    V0_2_10(Wrapper<v0_2_10::Version>),
+    V0_2_11(Wrapper<v0_2_11::Version>),
+    V0_2_12(Wrapper<v0_2_12::Version>),
+    V0_2_13(Wrapper<v0_2_13::Version>),
+    V0_2_14(Wrapper<v0_2_14::Version>),
+    V0_2_15(Wrapper<v0_2_15::Version>),
+    V0_2_16(Wrapper<v0_2_16::Version>),
     Other(emver::Version),
 }
 
@@ -149,6 +171,16 @@ pub async fn init() -> Result<(), failure::Error> {
             Version::V0_2_4(v) => v.0.migrate_to(&Current::new()).await?,
             Version::V0_2_5(v) => v.0.migrate_to(&Current::new()).await?,
             Version::V0_2_6(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_7(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_8(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_9(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_10(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_11(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_12(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_13(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_14(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_15(v) => v.0.migrate_to(&Current::new()).await?,
+            Version::V0_2_16(v) => v.0.migrate_to(&Current::new()).await?,
             Version::Other(_) => (),
             // TODO find some way to automate this?
         }
@@ -166,6 +198,7 @@ pub async fn self_update(requirement: emver::VersionRange) -> Result<(), Error>
     let url = format!("{}/appmgr?spec={}", &*crate::SYS_REGISTRY_URL, req_str);
     log::info!("Fetching new version from {}", url);
     let response = reqwest::get(&url)
+        .compat()
         .await
         .with_code(crate::error::NETWORK_ERROR)?
         .error_for_status()
@@ -235,6 +268,16 @@ pub async fn self_update(requirement: emver::VersionRange) -> Result<(), Error>
         Version::V0_2_4(v) => Current::new().migrate_to(&v.0).await?,
         Version::V0_2_5(v) => Current::new().migrate_to(&v.0).await?,
         Version::V0_2_6(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_7(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_8(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_9(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_10(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_11(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_12(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_13(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_14(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_15(v) => Current::new().migrate_to(&v.0).await?,
+        Version::V0_2_16(v) => Current::new().migrate_to(&v.0).await?,
         Version::Other(_) => (),
         // TODO find some way to automate this?
     };

appmgr/src/version/v0_1_0.rs

@@ -25,6 +25,7 @@ impl VersionT for Version {
         tokio::io::copy(
             &mut AsyncCompat(
                 reqwest::get(&format!("{}/torrc?spec==0.0.0", &*crate::SYS_REGISTRY_URL))
+                    .compat()
                     .await
                     .with_context(|e| format!("GET {}/torrc: {}", &*crate::SYS_REGISTRY_URL, e))
                     .with_code(crate::error::NETWORK_ERROR)?

appmgr/src/version/v0_1_1.rs

@@ -22,6 +22,7 @@ impl VersionT for Version {
         tokio::io::copy(
             &mut AsyncCompat(
                 reqwest::get(&format!("{}/torrc?spec==0.1.1", &*crate::SYS_REGISTRY_URL))
+                    .compat()
                     .await
                     .with_context(|e| format!("GET {}/torrc: {}", &*crate::SYS_REGISTRY_URL, e))
                     .with_code(crate::error::NETWORK_ERROR)?
@@ -76,6 +77,7 @@ impl VersionT for Version {
         tokio::io::copy(
             &mut AsyncCompat(
                 reqwest::get(&format!("{}/torrc?spec==0.1.0", &*crate::SYS_REGISTRY_URL))
+                    .compat()
                     .await
                     .with_context(|e| format!("GET {}/torrc: {}", &*crate::SYS_REGISTRY_URL, e))
                     .no_code()?

appmgr/src/version/v0_2_10.rs

@@ -0,0 +1,21 @@
+use super::*;
+
+const V0_2_10: emver::Version = emver::Version::new(0, 2, 10, 0);
+
+pub struct Version;
+#[async_trait]
+impl VersionT for Version {
+    type Previous = v0_2_9::Version;
+    fn new() -> Self {
+        Version
+    }
+    fn semver(&self) -> &'static emver::Version {
+        &V0_2_10
+    }
+    async fn up(&self) -> Result<(), Error> {
+        Ok(())
+    }
+    async fn down(&self) -> Result<(), Error> {
+        Ok(())
+    }
+}

appmgr/src/version/v0_2_11.rs

@@ -0,0 +1,38 @@
+use super::*;
+use std::os::unix::process::ExitStatusExt;
+
+const V0_2_11: emver::Version = emver::Version::new(0, 2, 11, 0);
+
+pub struct Version;
+#[async_trait]
+impl VersionT for Version {
+    type Previous = v0_2_10::Version;
+    fn new() -> Self {
+        Version
+    }
+    fn semver(&self) -> &'static emver::Version {
+        &V0_2_11
+    }
+    async fn up(&self) -> Result<(), Error> {
+        crate::tor::write_lan_services(
+            &crate::tor::services_map(&PersistencePath::from_ref(crate::SERVICES_YAML)).await?,
+        )
+        .await?;
+        let svc_exit = std::process::Command::new("service")
+            .args(&["nginx", "reload"])
+            .status()?;
+        crate::ensure_code!(
+            svc_exit.success(),
+            crate::error::GENERAL_ERROR,
+            "Failed to Reload Nginx: {}",
+            svc_exit
+                .code()
+                .or_else(|| { svc_exit.signal().map(|a| 128 + a) })
+                .unwrap_or(0)
+        );
+        Ok(())
+    }
+    async fn down(&self) -> Result<(), Error> {
+        Ok(())
+    }
+}

appmgr/src/version/v0_2_12.rs

@@ -0,0 +1,38 @@
+use super::*;
+use std::os::unix::process::ExitStatusExt;
+
+const V0_2_12: emver::Version = emver::Version::new(0, 2, 12, 0);
+
+pub struct Version;
+#[async_trait]
+impl VersionT for Version {
+    type Previous = v0_2_11::Version;
+    fn new() -> Self {
+        Version
+    }
+    fn semver(&self) -> &'static emver::Version {
+        &V0_2_12
+    }
+    async fn up(&self) -> Result<(), Error> {
+        crate::tor::write_lan_services(
+            &crate::tor::services_map(&PersistencePath::from_ref(crate::SERVICES_YAML)).await?,
+        )
+        .await?;
+        let svc_exit = std::process::Command::new("service")
+            .args(&["nginx", "reload"])
+            .status()?;
+        crate::ensure_code!(
+            svc_exit.success(),
+            crate::error::GENERAL_ERROR,
+            "Failed to Reload Nginx: {}",
+            svc_exit
+                .code()
+                .or_else(|| { svc_exit.signal().map(|a| 128 + a) })
+                .unwrap_or(0)
+        );
+        Ok(())
+    }
+    async fn down(&self) -> Result<(), Error> {
+        Ok(())
+    }
+}

appmgr/src/version/v0_2_13.rs

@@ -0,0 +1,21 @@
+use super::*;
+
+const V0_2_13: emver::Version = emver::Version::new(0, 2, 13, 0);
+
+pub struct Version;
+#[async_trait]
+impl VersionT for Version {
+    type Previous = v0_2_12::Version;
+    fn new() -> Self {
+        Version
+    }
+    fn semver(&self) -> &'static emver::Version {
+        &V0_2_13
+    }
+    async fn up(&self) -> Result<(), Error> {
+        Ok(())
+    }
+    async fn down(&self) -> Result<(), Error> {
+        Ok(())
+    }
+}

appmgr/src/version/v0_2_14.rs

@@ -0,0 +1,21 @@
+use super::*;
+
+const V0_2_14: emver::Version = emver::Version::new(0, 2, 14, 0);
+
+pub struct Version;
+#[async_trait]
+impl VersionT for Version {
+    type Previous = v0_2_13::Version;
+    fn new() -> Self {
+        Version
+    }
+    fn semver(&self) -> &'static emver::Version {
+        &V0_2_14
+    }
+    async fn up(&self) -> Result<(), Error> {
+        Ok(())
+    }
+    async fn down(&self) -> Result<(), Error> {
+        Ok(())
+    }
+}