From 748f9ec110966f98326e116b3f40651b6fcd8c83 Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Sat, 30 Apr 2022 07:07:09 -0400 Subject: [PATCH 01/13] Add initial draft of Matomo role --- roles/matomo/tasks/install.yml | 137 +++++++++++++++++++++++++++++++++ roles/matomo/tasks/main.yml | 9 +++ 2 files changed, 146 insertions(+) create mode 100644 roles/matomo/tasks/install.yml create mode 100644 roles/matomo/tasks/main.yml diff --git a/roles/matomo/tasks/install.yml b/roles/matomo/tasks/install.yml new file mode 100644 index 000000000..53b2838f1 --- /dev/null +++ b/roles/matomo/tasks/install.yml @@ -0,0 +1,137 @@ +- name: Start MariaDB + action: service name=mysql state=started +- name: Create MariaDB Database for Matomo + community.mysql.mysql_db: + name: "{{ mdb_dbname }}" + state: present + login_unix_socket: /var/run/mysqld/mysqld.sock +- name: Add Admin User to MariaDB Database + community.mysql.mysql_user: + name: "{{ mdb_username }}" + password: "{{ mdb_password }}" + host: localhost + state: present + update_password: on_create + priv: "{{ mdb_dbname }}.*:ALL" + login_unix_socket: /var/run/mysqld/mysqld.sock +- name: Download and Extract Matomo + unarchive: + src: https://builds.matomo.org/matomo.zip + dest: /var/www/html + remote_src: yes + creates: /var/www/html/matomo +- name: Set Matomo Directory Permissions + file: + path: /var/www/html/matomo + recurse: yes + owner: www-data + group: www-data +- name: HTTP Get Welcome + uri: + url: "{{ matomo_url }}index.php?action=welcome" + method: GET + status_code: 200 + register: matomo_welcome +- name: Set a variable for the MATOMO_SESSID cookie + set_fact: + matomo_session_cookie: "MATOMO_SESSID={{ cookie.value }}" + when: cookie.key == "MATOMO_SESSID" + loop: "{{ matomo_welcome.cookies | dict2items }}" + loop_control: + loop_var: cookie +- name: Get Matomo System Check + uri: + url: "{{matomo_url}}index.php?action=systemCheck" + method: GET + headers: + Cookie: "{{ matomo_session_cookie }}" + return_content: true + timeout: 120 + status_code: 200 + register: matomo_system_check +- name: Matomo Database Setup + uri: + url: "{{ matomo_url }}index.php?action=databaseSetup" + method: POST + headers: + Cookie: "{{ matomo_session_cookie }}" + body: + username: "{{ mdb_username }}" + password: "{{ mdb_password }}" + dbname: "{{ mdb_dbname }}" + adapter: "PDO\\MYSQL" + body_format: form-urlencoded + status_code: 302 + register: matomo_database_setup +- name: Matomo Table Creation + uri: + url: "{{ matomo_url }}index.php?action=tablesCreation&module=Installation" + method: GET + status_code: 200 + register: matomo_table_creation +- name: Matomo User Setup + uri: + url: "{{ matomo_url }}index.php?action=setupSuperUser&module=Installation" + method: POST + headers: + Cookie: "{{ matomo_session_cookie }}" + body: + login: "{{ mdb_username }}" + password: "{{ mdb_password }}" + password_bis: "{{ mdb_password }}" + e-mail: "nobody@dev.null" + subscribe_newsletter_piwikorg: 0 + subscribe_newsletter_professionalservices: 0 + body_format: form-urlencoded + status_code: 302 + register: matomo_setup_superuser +- name: Configure Matomo to track IIAB + uri: + url: "{{ matomo_url }}index.php?action=firstWebsiteSetup&module=Installation" + method: POST + headers: + Cookie: "{{ matomo_session_cookie }}" + body: + name: "IIAB" + url: "host_url" + timezone: "Europe/London" + ecommerce: 0 + body_format: form-urlencoded + status_code: 302 + register: matomo_first_website_setup +- name: Matomo Tracking Code + uri: + url: "{{ matomo_url }}index.php?action=trackingCode&module=Installation&site_idSite=1&site_name=http://10.0.0.72" + method: GET + headers: + Cookie: "{{ matomo_session_cookie }}" + return_content: true + status_code: 200 + register: matomo_tracking_code +- name: Finish Matomo Setup + uri: + url: "{{ matomo_url }}index.php?action=finished&module=Installation" + method: POST + headers: + Cookie: "{{ matomo_session_cookie }}" + body: + do_not_track: 1 + anonymise_ip: 1 + submit: "Continue to Matomo" + body_format: form-urlencoded + status_code: 302 +- name: Start Collecting Matomo Data + cron: + name: "MatomoDataIngestionOnReboot" + special_time: reboot + job: "{{ matomo_cronjob }}" + user: root + cron_file: "matomo_reboot" +- name: Run Daily Job Collecting Matomo Data + cron: + name: "DailyMatomoDataIngestion" + minute: "0" + hour: "0" + job: "{{ matomo_cronjob }}" + user: root + cron_file: "matomo_daily" diff --git a/roles/matomo/tasks/main.yml b/roles/matomo/tasks/main.yml new file mode 100644 index 000000000..a11d20b5f --- /dev/null +++ b/roles/matomo/tasks/main.yml @@ -0,0 +1,9 @@ +- name: Install Matomo main + include_tasks: install.yml + vars: + mdb_dbname: "matomodb" + mdb_username: "mariadb_admin" + mdb_password: "pw_set_by_ansible" + host_url: "http://{{ ansible_default_ipv4.address}}" + matomo_url: "{{ host_url }}/matomo/" + matomo_cronjob: "sudo python3 /library/www/html/matomo/misc/log-analytics/import_logs.py --url={{ matomo_url }} --idsite=1 --recorders=4 --enable-http-errors --enable-http-redirects --enable-static --enable-bots /var/log/nginx/access.log" From 37e750417d0a4d873a8924cfa707564382a78c28 Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Sun, 1 May 2022 06:09:49 -0400 Subject: [PATCH 02/13] Add citation for website interaction section. --- roles/matomo/tasks/install.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/roles/matomo/tasks/install.yml b/roles/matomo/tasks/install.yml index 53b2838f1..06fdaa9ed 100644 --- a/roles/matomo/tasks/install.yml +++ b/roles/matomo/tasks/install.yml @@ -1,3 +1,7 @@ +# The sections of code interacting with the Matomo website are modified from code found at https://git.coop/webarch/matomo/. This code is distributed under +# Version 3 of the GNU General Public License. We modified this code and applied it here in April 2022. The derived sections correspond to the tasks running +# from "HTTP Get Welcome" through "Finish Matomo Setup", lines 29 through 126. + - name: Start MariaDB action: service name=mysql state=started - name: Create MariaDB Database for Matomo From 1f2bd60002d74070d68d19ea1411bb7571a0ae50 Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Sun, 1 May 2022 06:10:39 -0400 Subject: [PATCH 03/13] Update password to IIAB default. --- roles/matomo/tasks/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matomo/tasks/main.yml b/roles/matomo/tasks/main.yml index a11d20b5f..77442e50e 100644 --- a/roles/matomo/tasks/main.yml +++ b/roles/matomo/tasks/main.yml @@ -2,8 +2,8 @@ include_tasks: install.yml vars: mdb_dbname: "matomodb" - mdb_username: "mariadb_admin" - mdb_password: "pw_set_by_ansible" + mdb_username: "iiab-admin" + mdb_password: "g0adm1n" host_url: "http://{{ ansible_default_ipv4.address}}" matomo_url: "{{ host_url }}/matomo/" matomo_cronjob: "sudo python3 /library/www/html/matomo/misc/log-analytics/import_logs.py --url={{ matomo_url }} --idsite=1 --recorders=4 --enable-http-errors --enable-http-redirects --enable-static --enable-bots /var/log/nginx/access.log" From 30620a803409e76a14f5a72b7f909fbc97cb58da Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Mon, 2 May 2022 06:24:50 -0400 Subject: [PATCH 04/13] changes to Matomo role to record state --- roles/matomo/tasks/install.yml | 8 ++++++++ roles/matomo/tasks/main.yml | 1 + 2 files changed, 9 insertions(+) diff --git a/roles/matomo/tasks/install.yml b/roles/matomo/tasks/install.yml index 06fdaa9ed..fdf8fa823 100644 --- a/roles/matomo/tasks/install.yml +++ b/roles/matomo/tasks/install.yml @@ -139,3 +139,11 @@ job: "{{ matomo_cronjob }}" user: root cron_file: "matomo_daily" +- name: Set Matomo state variable + set_fact: + matomo_installed: True +- name: Set Matomo state variable in IIAB state file + lineinfile: + path: "{{ iiab_state_file }}" + regexp: '^matomo_installed' + line: 'matomo_installed: True' diff --git a/roles/matomo/tasks/main.yml b/roles/matomo/tasks/main.yml index 77442e50e..36272833a 100644 --- a/roles/matomo/tasks/main.yml +++ b/roles/matomo/tasks/main.yml @@ -1,5 +1,6 @@ - name: Install Matomo main include_tasks: install.yml + when: matomo_installed is undefined vars: mdb_dbname: "matomodb" mdb_username: "iiab-admin" From 04fbec1f4b2da564b503d048c1617271676ba678 Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Mon, 2 May 2022 06:26:55 -0400 Subject: [PATCH 05/13] changes for matomo in mgmt tools --- roles/8-mgmt-tools/tasks/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/roles/8-mgmt-tools/tasks/main.yml b/roles/8-mgmt-tools/tasks/main.yml index e75f97e23..d9ea58dc1 100644 --- a/roles/8-mgmt-tools/tasks/main.yml +++ b/roles/8-mgmt-tools/tasks/main.yml @@ -13,6 +13,11 @@ name: awstats when: awstats_install +- name: MATOMO + include_role: + name: matomo + when: matomo_install + - name: MONIT include_role: name: monit From ba33825d6ad7d7a114e171b26b220eeaecf49e7e Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Tue, 3 May 2022 06:03:03 -0400 Subject: [PATCH 06/13] changes for matomo in validate vars --- roles/0-init/tasks/validate_vars.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/0-init/tasks/validate_vars.yml b/roles/0-init/tasks/validate_vars.yml index f29525daf..2ccd77860 100644 --- a/roles/0-init/tasks/validate_vars.yml +++ b/roles/0-init/tasks/validate_vars.yml @@ -111,6 +111,7 @@ - osm_vector_maps - transmission - awstats + - matomo - monit - munin - phpmyadmin From 45e1fd2c0efeb67fb1f9bddf6255a54eeb500488 Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Tue, 3 May 2022 06:33:15 -0400 Subject: [PATCH 07/13] correct matomo path and move to variable --- roles/matomo/tasks/install.yml | 4 ++-- roles/matomo/tasks/main.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/matomo/tasks/install.yml b/roles/matomo/tasks/install.yml index fdf8fa823..1fa5ab147 100644 --- a/roles/matomo/tasks/install.yml +++ b/roles/matomo/tasks/install.yml @@ -23,10 +23,10 @@ src: https://builds.matomo.org/matomo.zip dest: /var/www/html remote_src: yes - creates: /var/www/html/matomo + creates: "{{ matomo_nginx_loc }}" - name: Set Matomo Directory Permissions file: - path: /var/www/html/matomo + path: "{{ matomo_nginx_loc }}" recurse: yes owner: www-data group: www-data diff --git a/roles/matomo/tasks/main.yml b/roles/matomo/tasks/main.yml index 36272833a..b926afc3f 100644 --- a/roles/matomo/tasks/main.yml +++ b/roles/matomo/tasks/main.yml @@ -2,6 +2,7 @@ include_tasks: install.yml when: matomo_installed is undefined vars: + matomo_nginx_loc: "/library/www/html/matomo" mdb_dbname: "matomodb" mdb_username: "iiab-admin" mdb_password: "g0adm1n" From bd989dbbe2e6e2a18bd6e0d44e7696ee3057af40 Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Wed, 4 May 2022 06:25:16 -0400 Subject: [PATCH 08/13] Fix some bugs in paths/URLs --- roles/matomo/tasks/install.yml | 9 ++++----- roles/matomo/tasks/main.yml | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/roles/matomo/tasks/install.yml b/roles/matomo/tasks/install.yml index 1fa5ab147..5d8240905 100644 --- a/roles/matomo/tasks/install.yml +++ b/roles/matomo/tasks/install.yml @@ -21,12 +21,11 @@ - name: Download and Extract Matomo unarchive: src: https://builds.matomo.org/matomo.zip - dest: /var/www/html + dest: "{{ nginx_loc }}" remote_src: yes - creates: "{{ matomo_nginx_loc }}" - name: Set Matomo Directory Permissions file: - path: "{{ matomo_nginx_loc }}" + path: "{{ nginx_loc }}/matomo" recurse: yes owner: www-data group: www-data @@ -97,7 +96,7 @@ Cookie: "{{ matomo_session_cookie }}" body: name: "IIAB" - url: "host_url" + url: "{{ host_url }}" timezone: "Europe/London" ecommerce: 0 body_format: form-urlencoded @@ -105,7 +104,7 @@ register: matomo_first_website_setup - name: Matomo Tracking Code uri: - url: "{{ matomo_url }}index.php?action=trackingCode&module=Installation&site_idSite=1&site_name=http://10.0.0.72" + url: "{{ matomo_url }}index.php?action=trackingCode&module=Installation&site_idSite=1&site_name={{ host_url }}" method: GET headers: Cookie: "{{ matomo_session_cookie }}" diff --git a/roles/matomo/tasks/main.yml b/roles/matomo/tasks/main.yml index b926afc3f..799b171cf 100644 --- a/roles/matomo/tasks/main.yml +++ b/roles/matomo/tasks/main.yml @@ -2,7 +2,7 @@ include_tasks: install.yml when: matomo_installed is undefined vars: - matomo_nginx_loc: "/library/www/html/matomo" + nginx_loc: "/library/www/html" mdb_dbname: "matomodb" mdb_username: "iiab-admin" mdb_password: "g0adm1n" From d456ec3b919a0cde303ae8350c66780b982d603e Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Wed, 4 May 2022 06:29:21 -0400 Subject: [PATCH 09/13] revert breaking change in 0-init role --- roles/0-init/tasks/validate_vars.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/roles/0-init/tasks/validate_vars.yml b/roles/0-init/tasks/validate_vars.yml index 2ccd77860..f29525daf 100644 --- a/roles/0-init/tasks/validate_vars.yml +++ b/roles/0-init/tasks/validate_vars.yml @@ -111,7 +111,6 @@ - osm_vector_maps - transmission - awstats - - matomo - monit - munin - phpmyadmin From 3e9541ab7e937d6ec3917ec9dd3face0aca2944b Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Sat, 14 May 2022 14:19:14 -0400 Subject: [PATCH 10/13] Add missing field in Matomo cURL command --- roles/matomo/tasks/install.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/roles/matomo/tasks/install.yml b/roles/matomo/tasks/install.yml index 5d8240905..0a5684f4b 100644 --- a/roles/matomo/tasks/install.yml +++ b/roles/matomo/tasks/install.yml @@ -62,6 +62,7 @@ username: "{{ mdb_username }}" password: "{{ mdb_password }}" dbname: "{{ mdb_dbname }}" + tables_prefix: "matomo_" adapter: "PDO\\MYSQL" body_format: form-urlencoded status_code: 302 From fb326651a001b592dc04f6a035200a16922c128d Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Wed, 25 May 2022 06:51:24 -0400 Subject: [PATCH 11/13] Add section for cookie resetting --- roles/matomo/tasks/install.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/roles/matomo/tasks/install.yml b/roles/matomo/tasks/install.yml index 0a5684f4b..5c2303519 100644 --- a/roles/matomo/tasks/install.yml +++ b/roles/matomo/tasks/install.yml @@ -89,6 +89,16 @@ body_format: form-urlencoded status_code: 302 register: matomo_setup_superuser +- name: Set a variable for the MATOMO_SESSID cookie + set_fact: + matomo_session_cookie: "MATOMO_SESSID={{ cookie.value }}" + when: + - matomo_table_creation.cookies is defined + - matomo_table_creation.cookies | length > 0 + - cookie.key == "MATOMO_SESSID" + loop: "{{ matomo_table_creation.cookies | dict2items }}" + loop_control: + loop_var: cookie - name: Configure Matomo to track IIAB uri: url: "{{ matomo_url }}index.php?action=firstWebsiteSetup&module=Installation" From 4edc6c22bd7175e3c60a226b4f38e1a97b98bd2c Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Tue, 31 May 2022 06:51:57 -0400 Subject: [PATCH 12/13] Fix bug in cookie placement --- roles/matomo/tasks/install.yml | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/roles/matomo/tasks/install.yml b/roles/matomo/tasks/install.yml index 5c2303519..ac3aa0e9c 100644 --- a/roles/matomo/tasks/install.yml +++ b/roles/matomo/tasks/install.yml @@ -35,6 +35,10 @@ method: GET status_code: 200 register: matomo_welcome +- name: debug welcome + debug: + var: matomo_welcome + verbosity: 2 - name: Set a variable for the MATOMO_SESSID cookie set_fact: matomo_session_cookie: "MATOMO_SESSID={{ cookie.value }}" @@ -52,6 +56,10 @@ timeout: 120 status_code: 200 register: matomo_system_check +- name: debug syscheck + debug: + var: matomo_system_check + verbosity: 2 - name: Matomo Database Setup uri: url: "{{ matomo_url }}index.php?action=databaseSetup" @@ -73,6 +81,20 @@ method: GET status_code: 200 register: matomo_table_creation +- name: Set a variable for the MATOMO_SESSID cookie + set_fact: + matomo_session_cookie: "MATOMO_SESSID={{ cookie.value }}" + when: + - matomo_table_creation.cookies is defined + - matomo_table_creation.cookies | length > 0 + - cookie.key == "MATOMO_SESSID" + loop: "{{ matomo_table_creation.cookies | dict2items }}" + loop_control: + loop_var: cookie +- name: debug tablecreation + debug: + var: matomo_table_creation + verbosity: 2 - name: Matomo User Setup uri: url: "{{ matomo_url }}index.php?action=setupSuperUser&module=Installation" @@ -89,16 +111,6 @@ body_format: form-urlencoded status_code: 302 register: matomo_setup_superuser -- name: Set a variable for the MATOMO_SESSID cookie - set_fact: - matomo_session_cookie: "MATOMO_SESSID={{ cookie.value }}" - when: - - matomo_table_creation.cookies is defined - - matomo_table_creation.cookies | length > 0 - - cookie.key == "MATOMO_SESSID" - loop: "{{ matomo_table_creation.cookies | dict2items }}" - loop_control: - loop_var: cookie - name: Configure Matomo to track IIAB uri: url: "{{ matomo_url }}index.php?action=firstWebsiteSetup&module=Installation" From 0381225f531db6db3024fdef9ba2f7d760205868 Mon Sep 17 00:00:00 2001 From: Carl Wivagg Date: Sun, 12 Jun 2022 06:27:27 -0400 Subject: [PATCH 13/13] Fix remaining bugs --- roles/matomo/tasks/install.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/roles/matomo/tasks/install.yml b/roles/matomo/tasks/install.yml index ac3aa0e9c..4cad7236d 100644 --- a/roles/matomo/tasks/install.yml +++ b/roles/matomo/tasks/install.yml @@ -105,7 +105,7 @@ login: "{{ mdb_username }}" password: "{{ mdb_password }}" password_bis: "{{ mdb_password }}" - e-mail: "nobody@dev.null" + email: "nobody@dev.null" subscribe_newsletter_piwikorg: 0 subscribe_newsletter_professionalservices: 0 body_format: form-urlencoded @@ -118,7 +118,7 @@ headers: Cookie: "{{ matomo_session_cookie }}" body: - name: "IIAB" + siteName: "IIAB" url: "{{ host_url }}" timezone: "Europe/London" ecommerce: 0 @@ -161,11 +161,3 @@ job: "{{ matomo_cronjob }}" user: root cron_file: "matomo_daily" -- name: Set Matomo state variable - set_fact: - matomo_installed: True -- name: Set Matomo state variable in IIAB state file - lineinfile: - path: "{{ iiab_state_file }}" - regexp: '^matomo_installed' - line: 'matomo_installed: True'