Merge branch 'main' into feat/shadow-automation

Author: GrapeBaBa

README.md

@@ -54,7 +54,7 @@ validators:
       ip: "65.109.131.177"
       quic: 9001
     metricsPort: 9095
-    httpPort: 5055
+    apiPort: 5055
     isAggregator: false
     count: 1
 
@@ -97,27 +97,65 @@ validators:
     isAggregator: false
     count: 1
 
-  # - name: "gean_0"
+  - name: "gean_0"
+    # node id d8cfb453fbcbcf6bab2fe3d73d6b476101a813e40062e264a7173c79ad51f2b8
+    # peer id 16Uiu2HAmGbnwXEdBYDtARoSbkQ6wjtakg8EynnNaPcksatU5acXr
+    privkey: "df008e968231c25c3938d80fee9bcc93b4b9711312cf471c1b6f77e67ad68d08"
+    enrFields:
+      # verify /ip4/204.168.134.201/udp/9001/quic-v1/p2p/16Uiu2HAmGbnwXEdBYDtARoSbkQ6wjtakg8EynnNaPcksatU5acXr
+      ip: "204.168.134.201"
+      quic: 9001
+    metricsPort: 9095
+    apiPort: 5055
+    isAggregator: false
+    count: 1
+
+  - name: "nlean_0"
+    # node id 120f7715439bfdcb8025d26ade27488646298a4d9ac99f31ccf4f97fcc60fdfa
+    # peer id 16Uiu2HAmKyxGoN46ruQqSVzYyNUj2FKTWu7RGCLDGEdpzZ67EAde
+    privkey: "d94e3dc35e320440c891b66bd82d1aaf2079364162815b32c2633ecae009c84c"
+    enrFields:
+      # verify /ip4/95.216.164.165/udp/9001/quic-v1/p2p/16Uiu2HAmKyxGoN46ruQqSVzYyNUj2FKTWu7RGCLDGEdpzZ67EAde
+      ip: "95.216.164.165"
+      quic: 9001
+    metricsPort: 9095
+    apiPort: 5055
+    isAggregator: false
+    count: 1
+
+  # - name: "lean_node_0"
+  #   # node id 2289c0a43d51d4f50eee6101c580dbe3aba11eb4a9d32ba9993dd9976c5b994d
+  #   # peer id 16Uiu2HAkxUKxdny94t6WujGhVMBvpQJcTBVFGMGnzzkdyzbNLp7f
+  #   privkey: "520940446264167697a35be635041d651e48989ce2bb36698c444f42a2dd4f6c"
   #   enrFields:
-  #     ip: "204.168.134.201"
+  #     # verify /ip4/95.217.19.42/udp/9001/quic-v1/p2p/16Uiu2HAkxUKxdny94t6WujGhVMBvpQJcTBVFGMGnzzkdyzbNLp7f
+  #     ip: "95.217.19.42"
   #     quic: 9001
   #   metricsPort: 9095
   #   apiPort: 5055
   #   isAggregator: false
   #   count: 1
 
-  # - name: "lean_node_0"
+  # - name: "peam_0"
+  #   # node id 23537364cc6e2547f1ed53909e08e933c53aaed1c32a991e78cdcbc8e62c61cb
+  #   # peer id 16Uiu2HAmVSWJiHJMtfuLkyr8Du4BxTLg9rpgR32QHu1u3YuhcM3R
+  #   privkey: "8c7b62f4c4a35d134649817f44d2839ad15bcfa1fe6b87991f64d6f54f3ef2a1"
   #   enrFields:
-  #     ip: "95.217.19.42"
+  #     # verify /ip4/95.216.173.151/udp/9001/quic-v1/p2p/16Uiu2HAmVSWJiHJMtfuLkyr8Du4BxTLg9rpgR32QHu1u3YuhcM3R
+  #     ip: "95.216.173.151"
   #     quic: 9001
   #   metricsPort: 9095
   #   apiPort: 5055
   #   isAggregator: false
   #   count: 1
 
-  # - name: "peam_0"
+  # - name: "node_0"
+  #   # node id 4a27002fe8c79b68acf1e8af321fd4f3407492ab626de16174105234b4e726a7
+  #   # peer id 16Uiu2HAkxthAvokRQU71Secqos58q38TSXBqHHt5UUM5Viu2EuvF
+  #   privkey: "ebf06c49c66d4d08adaed41e901f49fd7439aa851110cf08c9ae54e9322a696e"
   #   enrFields:
-  #     ip: "95.216.173.151"
+  #     # verify /ip4/95.216.165.186/udp/9001/quic-v1/p2p/16Uiu2HAkxthAvokRQU71Secqos58q38TSXBqHHt5UUM5Viu2EuvF
+  #     ip: "95.216.165.186"
   #     quic: 9001
   #   metricsPort: 9095
   #   apiPort: 5055

ansible-devnet/genesis/validator-config.yaml

@@ -153,20 +153,30 @@
       loop_control:
         label: "{{ item }}.key"
 
+    - name: Resolve hash-sig key files for this node
+      vars:
+        _av: "{{ lookup('file', genesis_dir + '/annotated_validators.yaml') | from_yaml }}"
+        _assignments: "{{ _av[inventory_hostname] | default([]) }}"
+        _base: "{{ _assignments | map(attribute='privkey_file') | map('regex_replace', '_sk\\.ssz$', '') | list }}"
+      set_fact:
+        node_hash_sig_files: "{{ (_base | product(['_sk.ssz', '_pk.ssz', '_sk.json', '_pk.json']) | map('join') | list) + ['validator-keys-manifest.yaml'] }}"
+      when: hash_sig_keys_stat.stat.exists
+
     - name: Create hash-sig-keys directory on remote
       file:
         path: "{{ actual_remote_genesis_dir }}/hash-sig-keys"
         state: directory
         mode: '0755'
-      when: hash_sig_keys_stat.stat.exists
+      when: hash_sig_keys_stat.stat.exists and (node_hash_sig_files | default([]) | length > 0)
 
-    - name: Copy hash-sig-keys directory to remote host
+    - name: Copy hash-sig key files for this node only
       copy:
-        src: "{{ genesis_dir }}/hash-sig-keys/"
-        dest: "{{ actual_remote_genesis_dir }}/hash-sig-keys/"
-        mode: '0644'
+        src: "{{ genesis_dir }}/hash-sig-keys/{{ item }}"
+        dest: "{{ actual_remote_genesis_dir }}/hash-sig-keys/{{ item }}"
+        mode: '0600'
         force: yes
-      when: hash_sig_keys_stat.stat.exists
+      loop: "{{ node_hash_sig_files | default([]) }}"
+      when: hash_sig_keys_stat.stat.exists and (node_hash_sig_files | default([]) | length > 0)
 
     - name: List files on remote genesis directory
       find:

ansible/playbooks/copy-genesis.yml

@@ -8,7 +8,7 @@
 #   - node key files (*.key)
 #   - config.yaml, validators.yaml, nodes.yaml
 #   - genesis.ssz, genesis.json
-#   - hash-sig-keys/ directory (if exists, for qlean nodes)
+#   - hash-sig-keys/ directory (if exists): only the sk/pk files for this node's validators
 
 - name: Parse and validate node names
   hosts: localhost
@@ -122,7 +122,10 @@
 
     - name: Sync validator-config.yaml to remote host
       copy:
-        src: "{{ local_genesis_dir }}/validator-config.yaml"
+        # Use the expanded subnet config when --subnets was specified; fall back
+        # to the standard validator-config.yaml otherwise. The destination is
+        # always validator-config.yaml so client roles don't need to change.
+        src: "{{ local_genesis_dir }}/{{ validator_config_basename | default('validator-config.yaml') }}"
         dest: "{{ genesis_dir }}/validator-config.yaml"
         mode: '0644'
         force: yes
@@ -149,6 +152,7 @@
       loop:
         - config.yaml
         - validators.yaml
+        - annotated_validators.yaml
         - nodes.yaml
         - genesis.ssz
         - genesis.json
@@ -165,23 +169,36 @@
         - deploy
         - sync
 
+    - name: Resolve hash-sig key files for this node
+      vars:
+        _av: "{{ lookup('file', local_genesis_dir + '/annotated_validators.yaml') | from_yaml }}"
+        _assignments: "{{ _av[node_name] | default([]) }}"
+        _base: "{{ _assignments | map(attribute='privkey_file') | map('regex_replace', '_sk\\.ssz$', '') | list }}"
+      set_fact:
+        node_hash_sig_files: "{{ (_base | product(['_sk.ssz', '_pk.ssz', '_sk.json', '_pk.json']) | map('join') | list) + ['validator-keys-manifest.yaml'] }}"
+      when: hash_sig_keys_local.stat.exists
+      tags:
+        - deploy
+        - sync
+
     - name: Create hash-sig-keys directory on remote
       file:
         path: "{{ genesis_dir }}/hash-sig-keys"
         state: directory
         mode: '0755'
-      when: hash_sig_keys_local.stat.exists
+      when: hash_sig_keys_local.stat.exists and (node_hash_sig_files | default([]) | length > 0)
       tags:
         - deploy
         - sync
 
-    - name: Sync hash-sig-keys directory (for qlean nodes)
+    - name: Copy hash-sig key files for this node only
       copy:
-        src: "{{ local_genesis_dir }}/hash-sig-keys/"
-        dest: "{{ genesis_dir }}/hash-sig-keys/"
-        mode: '0644'
+        src: "{{ local_genesis_dir }}/hash-sig-keys/{{ item }}"
+        dest: "{{ genesis_dir }}/hash-sig-keys/{{ item }}"
+        mode: '0600'
         force: yes
-      when: hash_sig_keys_local.stat.exists
+      loop: "{{ node_hash_sig_files | default([]) }}"
+      when: hash_sig_keys_local.stat.exists and (node_hash_sig_files | default([]) | length > 0)
       tags:
         - deploy
         - sync
@@ -199,6 +216,9 @@
         - lighthouse
         - grandine
         - ethlambda
+        - gean
+        - nlean
+        - peam
         - deploy
 
     - name: Include common role
@@ -224,3 +244,37 @@
       tags:
         - deploy
 
+# When --replace-with is used, sync updated config yamls to all remote hosts
+- name: Sync updated config files to all hosts after replacement
+  hosts: all:!local
+  gather_facts: no
+  vars:
+    genesis_dir: "{{ remote_genesis_dir | default('/opt/lean-quickstart/genesis') }}"
+    local_genesis_dir: "{{ hostvars['localhost']['local_genesis_dir_path'] }}"
+  tasks:
+    - name: Sync config yamls to all hosts
+      copy:
+        src: "{{ local_genesis_dir }}/{{ item }}"
+        dest: "{{ genesis_dir }}/{{ item }}"
+        mode: '0644'
+        force: yes
+      loop:
+        - validators.yaml
+        - annotated_validators.yaml
+        - nodes.yaml
+        - validator-config.yaml
+      when: sync_all_hosts | default(false) | bool
+      tags:
+        - deploy
+        - sync
+
+    - name: Sync validator-keys-manifest to all hosts
+      copy:
+        src: "{{ local_genesis_dir }}/hash-sig-keys/validator-keys-manifest.yaml"
+        dest: "{{ genesis_dir }}/hash-sig-keys/validator-keys-manifest.yaml"
+        mode: '0644'
+        force: yes
+      when: sync_all_hosts | default(false) | bool
+      tags:
+        - deploy
+        - sync

ansible/playbooks/deploy-nodes.yml

@@ -86,7 +86,31 @@
     - ethlambda
     - deploy
 
+- name: Deploy Gean node
+  include_role:
+    name: gean
+  when: client_type == "gean"
+  tags:
+    - gean
+    - deploy
+
+- name: Deploy Nlean node
+  include_role:
+    name: nlean
+  when: client_type == "nlean"
+  tags:
+    - nlean
+    - deploy
+
+- name: Deploy Peam node
+  include_role:
+    name: peam
+  when: client_type == "peam"
+  tags:
+    - peam
+    - deploy
+
 - name: Fail if unknown client type
   fail:
-    msg: "Unknown client type '{{ client_type }}' for node '{{ node_name }}'. Expected: zeam, ream, qlean, lantern, lighthouse, grandine or ethlambda"
-  when: client_type not in ["zeam", "ream", "qlean", "lantern", "lighthouse", "grandine", "ethlambda"]
+    msg: "Unknown client type '{{ client_type }}' for node '{{ node_name }}'. Expected: zeam, ream, qlean, lantern, lighthouse, grandine, ethlambda, gean, nlean or peam"
+  when: client_type not in ["zeam", "ream", "qlean", "lantern", "lighthouse", "grandine", "ethlambda", "gean", "nlean", "peam"]