Internet Archive installation (merged from Georgejhunt & mitra42 distweb branches)

roles/9-local-addons/tasks/main.yml

@@ -21,6 +21,12 @@
   when: minetest_install
   tags: minetest
 
+- name: INTERNETARCHIVE
+  include_role:
+    name: internetarchive
+  when: internetarchive_install
+  tags: internetarchive
+
 - name: Recording STAGE 9 HAS COMPLETED ====================
   lineinfile:
     dest: "{{ iiab_env_file }}"

roles/internetarchive/defaults/main.yml

@@ -0,0 +1,7 @@
+internetarchive_port: 4244
+internetarchive_dir: '{{ iiab_base }}/internetarchive'
+# for testing purposes, install and enable
+internetarchive_install: True
+internetarchive_enabled: True
+#internetarchive_install: False
+#internetarchive_enabled: False

roles/internetarchive/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+    - { role: nodejs, tags: ['nodejs'], when: internetarchive_install }
+    - { role: yarn, tags: ['yarn'], when: internetarchive_install }

roles/internetarchive/tasks/main.yml

@@ -0,0 +1,98 @@
+# We need a recent version of node
+
+- name: FAIL (STOP INSTALLING) IF nodejs_version is not set to 10.x
+  fail:
+    msg: "Internet Archive install cannot proceeed, as it currently requires Node.js 10.x, and your nodejs_version is set to {{ nodejs_version }}.  Please check the value of nodejs_version in /opt/iiab/iiab/vars/default_vars.yml and possibly also /etc/iiab/local_vars.yml"
+  when: internetarchive_install and (nodejs_version != "10.x")
+
+- name: Install packages needed by Distributed Web
+  package:
+      name:
+         - libsecret-1-dev
+         - cmake
+      state: present
+
+- name: Create directory {{ internetarchive_dir }}
+  file:
+    path: "{{ internetarchive_dir }}"
+    state: directory
+    owner: "root"
+
+- name: Run yarn install to get needed modules (CAN TAKE ~5 MINUTES)
+  command: sudo yarn add @internetarchive/dweb-archive @internetarchive/dweb-mirror
+  args:
+    chdir: "{{ internetarchive_dir }}"
+  when: internet_available
+
+- name: Create directory /library/archiveorg
+  file:
+    path: "/library/archiveorg"
+    state: directory
+    owner: "root"
+
+# CONFIG FILES
+
+- name: "Install from templates: internetarchive.service (systemd), internetarchive.conf (Apache)"
+  template:
+    src: "{{ item.src }}"
+    dest: "{{ item.dest }}"
+    mode: 0644
+    owner: root
+    group: root
+  with_items:
+    - { src: 'internetarchive.service.j2', dest: '/etc/systemd/system/internetarchive.service' }
+    - { src: 'internetarchive.conf', dest: '/etc/apache2/sites-available/internetarchive.conf' }
+
+
+- name: Create symlink internetarchive.conf from sites-enabled to sites-available, for short URLs http://box/sugar & http://box/internetarchive (if internetarchive_enabled)
+  file:
+    src: /etc/apache2/sites-available/internetarchive.conf
+    path: /etc/apache2/sites-enabled/internetarchive.conf
+    state: link
+  when: internetarchive_enabled and is_debuntu
+
+- name: Remove symlink /etc/apache2/sites-enabled/internetarchive.conf (if not internetarchive_enabled)
+  file:
+    path: /etc/apache2/sites-enabled/internetarchive.conf
+    state: absent
+  when: not internetarchive_enabled and is_debuntu
+
+
+# RESTART/STOP SYSTEMD SERVICE
+
+# with "systemctl daemon-reload" in case mongodb.service changed, etc
+- name: Enable & Restart 'internetarchive' systemd service (if internetarchive_enabled)
+  systemd:
+    name: internetarchive
+    daemon_reload: yes
+    enabled: yes
+    state: restarted
+  when: internetarchive_enabled
+  
+- name: Disable & Stop 'internetarchive' systemd service (if not internetarchive_enabled)
+  systemd:
+    name: internetarchive
+    daemon_reload: yes
+    enabled: no
+    state: stopped
+  when: not internetarchive_enabled
+
+- name: Restart Apache service ({{ apache_service }}) to enable/disable http://box/internetarchive (not just http://box:{{ internetarchive_port }})
+  systemd:
+    name: "{{ apache_service }}"    # httpd or apache2
+    state: restarted
+  when: internetarchive_enabled
+
+- name: Add 'internetarchive' variable values to {{ iiab_ini_file }}
+  ini_file:
+    path: "{{ iiab_ini_file }}"
+    section: internetarchive
+    option: "{{ item.option }}"
+    value: "{{ item.value }}"
+  with_items:
+    - option: name
+      value: Archive Org Distributed Web
+    - option: description
+      value: '"Dweb-mirror is intended to make the Internet Archive experience and UI available offline."'
+    - option: internetarchive_enabled
+      value: "{{ internetarchive_enabled }}"

roles/internetarchive/templates/internetarchive.conf

@@ -0,0 +1,8 @@
+# internetarchive_port is set to 4244 in /roles/archoeg/defaults/main.yml
+# If you need to change this, edit /etc/iiab/local_vars.yml prior to installing
+
+RedirectMatch ^/archive.org$ /internetarchive
+RedirectMatch ^/archive$ /internetarchive
+
+ProxyPass /internetarchive http://localhost:{{ internetarchive_port }}/internetarchive
+ProxyPassReverse /internetarchive http://localhost:{{ internetarchive_port }}/internetarchive

roles/internetarchive/templates/internetarchive.service.j2

@@ -0,0 +1,17 @@
+[Unit]
+Description=Internet Archive Universal Library service
+After=network-online.target
+
+[Service]
+Type=simple
+WorkingDirectory={{ internetarchive_dir }}/node_modules/@internetarchive/dweb-mirror
+ExecStart=/usr/bin/node ./internetarchive -s
+Restart=always
+RestartSec=10
+StandardOutput=syslog
+StandardError=syslog
+SyslogIdentifier=internetarchive
+
+[Install]
+WantedBy=multi-user.target
+

roles/network/templates/gateway/iiab-gen-iptables

@@ -63,6 +63,7 @@ cups_port={{ cups_port }}
 transmission_http_port={{ transmission_http_port }}
 transmission_peer_port={{ transmission_peer_port }}
 sugarizer_port={{ sugarizer_port }}
+internetarchive_port={{ internetarchive_port }}
 nodered_port={{ nodered_port }}
 mosquitto_port={{ mosquitto_port }}
 minetest_port={{ minetest_port }}
@@ -105,6 +106,7 @@ if [ "$services_externally_visible" == "True" ]; then
     $IPTABLES -A INPUT -p tcp --dport $calibreweb_port -m state --state NEW -i $wan -j ACCEPT
     $IPTABLES -A INPUT -p tcp --dport $cups_port -m state --state NEW -i $wan -j ACCEPT
     $IPTABLES -A INPUT -p tcp --dport $sugarizer_port -m state --state NEW -i $wan -j ACCEPT
+    $IPTABLES -A INPUT -p tcp --dport $internetarchive_port -m state --state NEW -i $wan -j ACCEPT
     $IPTABLES -A INPUT -p tcp --dport $nodered_port -m state --state NEW -i $wan -j ACCEPT
     $IPTABLES -A INPUT -p tcp --dport $mosquitto_port -m state --state NEW -i $wan -j ACCEPT
     $IPTABLES -A INPUT -p tcp --dport $transmission_http_port -m state --state NEW -i $wan -j ACCEPT

roles/yarn/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+
+- name: "Yarn | GPG"
+  apt_key:
+    url: https://dl.yarnpkg.com/debian/pubkey.gpg
+    state: present
+
+- name: "Yarn | Ensure Debian sources list file exists"
+  file:
+    path: /etc/apt/sources.list.d/yarn.list
+    owner: root
+    mode: 0644
+    state: touch
+
+- name: "Yarn | Ensure Debian package is in sources list"
+  lineinfile:
+    dest: /etc/apt/sources.list.d/yarn.list
+    regexp: 'deb http://dl.yarnpkg.com/debian/ stable main'
+    line: 'deb http://dl.yarnpkg.com/debian/ stable main'
+    state: present
+
+- name: "Yarn | Update APT cache"
+  apt:
+    update_cache: yes
+
+- name: "Yarn | Install"
+  package:
+    name: yarn
+    state: latest
+  when: internet_available and is_debuntu

vars/default_vars.yml

@@ -459,6 +459,12 @@ calibreweb_port: 8083    # PORT VARIABLE HAS NO EFFECT (as of January 2019)
 calibreweb_url: /books
 calibreweb_home: "{{ content_base }}/calibre-web"    # /library/calibre-web
 
+#InternetArchive Dweb service - port needs wider scope
+internetarchive_port: 4244
+# Added here, because noone is clear where they SHOULD go
+internetarchive_install: True
+internetarchive_enabled: True
+
 # Minetest is an open source clone of the Minecraft building blocks game
 minetest_install: False
 minetest_enabled: False
@@ -467,7 +473,6 @@ minetest_server_admin: Admin
 minetest_default_game: carbone-ng    # only carbone-ng and minetest are supported
 minetest_flat_world: False
 
-
 # CONSIDER THESE 2 NEW OPENSTREETMAP (OSM) APPROACHES INSTEAD, AS OF 2018:
 # - http://download.iiab.io/content/OSM/vector-tiles/
 # - http://oer2go.org/viewmod/en-worldmap-10