refactor(infra): restructure inventory and update dependencies

- rename manager node and update host FQDNs in inventory
- enable multiple worker nodes by commenting out secondary disk
- standardize k8s configuration file paths with numeric prefixes
- disable k9s permissions step in manager role
- upgrade Longhorn chart to v1.12.0 and Traefik chart to v41.0.2
- update Traefik port_map with protocol and add forgejo service
- refine Traefik logging configuration to use new JSON format
- increase Longhorn disk probe retries and delays
- add placeholder directory for k8s_manager files

inventory/prod/group_vars/traefik.yml

@@ -1,31 +1,45 @@
 ---
-traefik_enable_ingress: true
+#traefik_enable_ingress: true
-traefik_ingress_class_name: "traefik"
+#traefik_ingress_class_name: "traefik"
-traefik_namespace: kube-system
+#traefik_namespace: kube-system
 
 traefik_port_map:
 
   - name: k8s-dashboard
     port: 10001
+    protocol: tcp
     backend:
       namespace: kubernetes-dashboard
       service: kubernetes-dashboard
       port: 443
-      scheme: https
+#      scheme: https
     basicauth:
       enabled: false
 
   - name: longhorn-ui
     port: 10002
+    protocol: tcp
     backend:
       namespace: longhorn-system
       service: longhorn-frontend
       port: 80
-      scheme: http
+#      scheme: http
     basicauth:
       enabled: true
       secret_name: traefik-auth-longhorn
 
+  - name: forgejo
+    port: 10003
+    protocol: tcp
+    backend:
+      namespace: git
+      service: forgejo-http
+      port: 3000
+#      scheme: http
+    basicauth:
+      enabled: false
+
+
   # Template for future services:
   # - name: my-service
   #   description: "Human-readable description"

inventory/prod/group_vars/workers.yml

@@ -8,5 +8,5 @@
 longhorn_disks:
   - device: /dev/sdb
     mountpoint: /mnt/longhorn-disk1
-  - device: /dev/sdc
+#  - device: /dev/sdc
-    mountpoint: /mnt/longhorn-disk2
+#    mountpoint: /mnt/longhorn-disk2

inventory/prod/hosts.yml

@@ -4,20 +4,20 @@ all:
     # Operator workstation: kubectl, helm, k9s
     manager_nodes:
       hosts:
-        k8s-manager-01:
+        mgmt.local.mrwho.ru:
-          ansible_host: 172.20.21.200
+          ansible_host: 172.20.21.199
 
     # Kubernetes cluster nodes
     k8s_cluster:
       children:
         control_plane:
           hosts:
-            k8s-master-01:
+            k8s-master-01.local.mrwho.ru:
-              ansible_host: 172.20.21.201
+              ansible_host: 172.20.21.200
         workers:
           hosts:
-            k8s-worker-01:
+            k8s-worker-01.local.mrwho.ru:
-              ansible_host: 172.20.21.202
+              ansible_host: 172.20.21.201
               longhorn_disks:
                 - device: /dev/sdb
                   mountpoint: /mnt/longhorn-disk1

roles/k8s_control_plane/tasks/prerequisites.yml

@@ -27,7 +27,7 @@
 
 - name: Prerequisites | Load kernel modules (persistent)
   ansible.builtin.copy:
-    dest: /etc/modules-load.d/k8s.conf
+    dest: /etc/modules-load.d/99-k8s.conf
     content: |
       overlay
       br_netfilter
@@ -45,7 +45,7 @@
   ansible.posix.sysctl:
     name: "{{ item.key }}"
     value: "{{ item.value }}"
-    sysctl_file: /etc/sysctl.d/k8s.conf
+    sysctl_file: /etc/sysctl.d/99-k8s.conf
     reload: true
   loop:
     - { key: "net.bridge.bridge-nf-call-iptables", value: "1" }

roles/k8s_manager/files/kubectx

@@ -0,0 +1,254 @@
+#!/usr/bin/env bash
+#
+# kubectx(1) is a utility to manage and switch between kubectl contexts.
+
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[[ -n $DEBUG ]] && set -x
+
+set -eou pipefail
+IFS=$'\n\t'
+
+SELF_CMD="$0"
+
+KUBECTX="${XDG_CACHE_HOME:-$HOME/.kube}/kubectx"
+
+usage() {
+  local SELF
+  SELF="kubectx"
+  if [[ "$(basename "$0")" == kubectl-* ]]; then # invoked as plugin
+    SELF="kubectl ctx"
+  fi
+
+  cat <<EOF
+Manage and switch between kubectl contexts.
+
+USAGE:
+  $SELF                       : list the contexts
+  $SELF <NAME>                : switch to context <NAME>
+  $SELF -                     : switch to the previous context
+  $SELF -c, --current         : show the current context name
+  $SELF <NEW_NAME>=<NAME>     : rename context <NAME> to <NEW_NAME>
+  $SELF <NEW_NAME>=.          : rename current-context to <NEW_NAME>
+  $SELF -d <NAME> [<NAME...>] : delete context <NAME> ('.' for current-context)
+                                  (this command won't delete the user/cluster entry
+                                  that is used by the context)
+  $SELF -u, --unset           : unset the current context
+
+  $SELF -h,--help             : show this message
+
+  (This executable is the legacy bash-based implementation, consider upgrading to Go-based implementation.)
+EOF
+}
+
+exit_err() {
+   echo >&2 "${1}"
+   exit 1
+}
+
+current_context() {
+  $KUBECTL config view -o=jsonpath='{.current-context}'
+}
+
+get_contexts() {
+  $KUBECTL config get-contexts -o=name | sort -n
+}
+
+list_contexts() {
+  set -u pipefail
+  local cur ctx_list
+  cur="$(current_context)" || exit_err "error getting current context"
+  ctx_list=$(get_contexts) || exit_err "error getting context list"
+
+  local yellow darkbg normal
+  yellow=$(tput setaf 3 || true)
+  darkbg=$(tput setab 0 || true)
+  normal=$(tput sgr0 || true)
+
+  local cur_ctx_fg cur_ctx_bg
+  cur_ctx_fg=${KUBECTX_CURRENT_FGCOLOR:-$yellow}
+  cur_ctx_bg=${KUBECTX_CURRENT_BGCOLOR:-$darkbg}
+
+  for c in $ctx_list; do
+  if [[ -n "${_KUBECTX_FORCE_COLOR:-}" || \
+       -t 1 && -z "${NO_COLOR:-}" ]]; then
+    # colored output mode
+    if [[ "${c}" = "${cur}" ]]; then
+      echo "${cur_ctx_bg}${cur_ctx_fg}${c}${normal}"
+    else
+      echo "${c}"
+    fi
+  else
+    echo "${c}"
+  fi
+  done
+}
+
+read_context() {
+  if [[ -f "${KUBECTX}" ]]; then
+    cat "${KUBECTX}"
+  fi
+}
+
+save_context() {
+  local saved
+  saved="$(read_context)"
+
+  if [[ "${saved}" != "${1}" ]]; then
+    printf %s "${1}" > "${KUBECTX}"
+  fi
+}
+
+switch_context() {
+  $KUBECTL config use-context "${1}"
+}
+
+choose_context_interactive() {
+  local choice
+  choice="$(_KUBECTX_FORCE_COLOR=1 \
+    FZF_DEFAULT_COMMAND="${SELF_CMD}" \
+    fzf --ansi --no-preview || true)"
+  if [[ -z "${choice}" ]]; then
+    echo 2>&1 "error: you did not choose any of the options"
+    exit 1
+  else
+    set_context "${choice}"
+  fi
+}
+
+set_context() {
+  local prev
+  prev="$(current_context)" || exit_err "error getting current context"
+
+  switch_context "${1}"
+
+  if [[ "${prev}" != "${1}" ]]; then
+    save_context "${prev}"
+  fi
+}
+
+swap_context() {
+  local ctx
+  ctx="$(read_context)"
+  if [[ -z "${ctx}" ]]; then
+    echo "error: No previous context found." >&2
+    exit 1
+  fi
+  set_context "${ctx}"
+}
+
+context_exists() {
+  grep -q ^"${1}"\$ <($KUBECTL config get-contexts -o=name)
+}
+
+rename_context() {
+  local old_name="${1}"
+  local new_name="${2}"
+
+  if [[ "${old_name}" == "." ]]; then
+    old_name="$(current_context)"
+  fi
+
+  if ! context_exists "${old_name}"; then
+    echo "error: Context \"${old_name}\" not found, can't rename it." >&2
+    exit 1
+  fi
+
+  if context_exists "${new_name}"; then
+    echo "Context \"${new_name}\" exists, deleting..." >&2
+    $KUBECTL config delete-context "${new_name}" 1>/dev/null 2>&1
+  fi
+
+  $KUBECTL config rename-context "${old_name}" "${new_name}"
+}
+
+delete_contexts() {
+  for i in "${@}"; do
+    delete_context "${i}"
+  done
+}
+
+delete_context() {
+  local ctx
+  ctx="${1}"
+  if [[ "${ctx}" == "." ]]; then
+    ctx="$(current_context)" || exit_err "error getting current context"
+  fi
+  echo "Deleting context \"${ctx}\"..." >&2
+  $KUBECTL config delete-context "${ctx}"
+}
+
+unset_context() {
+  echo "Unsetting current context." >&2
+  $KUBECTL config unset current-context
+}
+
+main() {
+  if [[ -z "${KUBECTL:-}" ]]; then
+    if hash kubectl 2>/dev/null; then
+      KUBECTL=kubectl
+    elif hash kubectl.exe  2>/dev/null; then
+      KUBECTL=kubectl.exe
+    else
+      echo >&2 "kubectl is not installed"
+      exit 1
+    fi
+  fi
+
+  if [[ "$#" -eq 0 ]]; then
+    if [[ -t 1 &&  -z "${KUBECTX_IGNORE_FZF:-}" && "$(type fzf &>/dev/null; echo $?)" -eq 0 ]]; then
+      choose_context_interactive
+    else
+      list_contexts
+    fi
+  elif [[ "${1}" == "-d" ]]; then
+    if [[ "$#" -lt 2 ]]; then
+      echo "error: missing context NAME" >&2
+      usage
+      exit 1
+    fi
+    delete_contexts "${@:2}"
+  elif [[ "$#" -gt 1 ]]; then
+    echo "error: too many arguments" >&2
+    usage
+    exit 1
+  elif [[ "$#" -eq 1 ]]; then
+    if [[ "${1}" == "-" ]]; then
+      swap_context
+    elif [[ "${1}" == '-c' || "${1}" == '--current' ]]; then
+      # we don't call current_context here for two reasons:
+      # - it does not fail when current-context property is not set
+      # - it does not return a trailing newline
+      $KUBECTL config current-context
+    elif [[ "${1}" == '-u' || "${1}" == '--unset' ]]; then
+      unset_context
+    elif [[ "${1}" == '-h' || "${1}" == '--help' ]]; then
+      usage
+    elif [[ "${1}" =~ ^-(.*) ]]; then
+      echo "error: unrecognized flag \"${1}\"" >&2
+      usage
+      exit 1
+    elif [[ "${1}" =~ (.+)=(.+) ]]; then
+      rename_context "${BASH_REMATCH[2]}" "${BASH_REMATCH[1]}"
+    else
+      set_context "${1}"
+    fi
+  else
+    usage
+    exit 1
+  fi
+}
+
+main "$@"

roles/k8s_manager/files/kubens

@@ -0,0 +1,230 @@
+#!/usr/bin/env bash
+#
+# kubens(1) is a utility to switch between Kubernetes namespaces.
+
+# Copyright 2017 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[[ -n $DEBUG ]] && set -x
+
+set -eou pipefail
+IFS=$'\n\t'
+
+SELF_CMD="$0"
+
+KUBENS_DIR="${XDG_CACHE_HOME:-$HOME/.kube}/kubens"
+
+usage() {
+  local SELF
+  SELF="kubens"
+  if [[ "$(basename "$0")" == kubectl-* ]]; then # invoked as plugin
+    SELF="kubectl ns"
+  fi
+
+  cat <<EOF
+Switch between Kubernetes namespaces.
+
+USAGE:
+  $SELF                    : list the namespaces in the current context
+  $SELF <NAME>             : change the active namespace of current context
+  $SELF -                  : switch to the previous namespace in this context
+  $SELF -c, --current      : show the current namespace
+  $SELF -h,--help          : show this message
+
+  (This executable is the legacy bash-based implementation, consider upgrading to Go-based implementation.)
+EOF
+}
+
+exit_err() {
+   echo >&2 "${1}"
+   exit 1
+}
+
+current_namespace() {
+  local cur_ctx
+
+  cur_ctx="$(current_context)" || exit_err "error getting current context"
+  ns="$($KUBECTL config view -o=jsonpath="{.contexts[?(@.name==\"${cur_ctx}\")].context.namespace}")" \
+     || exit_err "error getting current namespace"
+
+  if [[ -z "${ns}" ]]; then
+    echo "default"
+  else
+    echo "${ns}"
+  fi
+}
+
+current_context() {
+  $KUBECTL config current-context
+}
+
+get_namespaces() {
+  $KUBECTL get namespaces -o=jsonpath='{range .items[*].metadata.name}{@}{"\n"}{end}'
+}
+
+escape_context_name() {
+  echo "${1//\//-}"
+}
+
+namespace_file() {
+  local ctx
+
+  ctx="$(escape_context_name "${1}")"
+  echo "${KUBENS_DIR}/${ctx}"
+}
+
+read_namespace() {
+  local f
+  f="$(namespace_file "${1}")"
+  [[ -f "${f}" ]] && cat "${f}"
+  return 0
+}
+
+save_namespace() {
+  mkdir -p "${KUBENS_DIR}"
+  local f saved
+  f="$(namespace_file "${1}")"
+  saved="$(read_namespace "${1}")"
+
+  if [[ "${saved}" != "${2}" ]]; then
+    printf %s "${2}" > "${f}"
+  fi
+}
+
+switch_namespace() {
+  local ctx="${1}"
+  $KUBECTL config set-context "${ctx}" --namespace="${2}"
+  echo "Active namespace is \"${2}\".">&2
+}
+
+choose_namespace_interactive() {
+  # directly calling kubens via fzf might fail with a cryptic error like
+  # "$FZF_DEFAULT_COMMAND failed", so try to see if we can list namespaces
+  # locally first
+  if [[ -z "$(list_namespaces)" ]]; then
+    echo >&2 "error: could not list namespaces (is the cluster accessible?)"
+    exit 1
+  fi
+
+  local choice
+  choice="$(_KUBECTX_FORCE_COLOR=1 \
+    FZF_DEFAULT_COMMAND="${SELF_CMD}" \
+    fzf --ansi --no-preview || true)"
+  if [[ -z "${choice}" ]]; then
+    echo 2>&1 "error: you did not choose any of the options"
+    exit 1
+  else
+    set_namespace "${choice}"
+  fi
+}
+
+set_namespace() {
+  local ctx prev
+  ctx="$(current_context)" || exit_err "error getting current context"
+  prev="$(current_namespace)" || exit_error "error getting current namespace"
+
+  if grep -q ^"${1}"\$ <(get_namespaces); then
+    switch_namespace "${ctx}" "${1}"
+
+    if [[ "${prev}" != "${1}" ]]; then
+      save_namespace "${ctx}" "${prev}"
+    fi
+  else
+    echo "error: no namespace exists with name \"${1}\".">&2
+    exit 1
+  fi
+}
+
+list_namespaces() {
+  local yellow darkbg normal
+  yellow=$(tput setaf 3 || true)
+  darkbg=$(tput setab 0 || true)
+  normal=$(tput sgr0 || true)
+
+  local cur_ctx_fg cur_ctx_bg
+  cur_ctx_fg=${KUBECTX_CURRENT_FGCOLOR:-$yellow}
+  cur_ctx_bg=${KUBECTX_CURRENT_BGCOLOR:-$darkbg}
+
+  local cur ns_list
+  cur="$(current_namespace)" || exit_err "error getting current namespace"
+  ns_list=$(get_namespaces) || exit_err "error getting namespace list"
+
+  for c in $ns_list; do
+  if [[ -n "${_KUBECTX_FORCE_COLOR:-}" || \
+       -t 1 && -z "${NO_COLOR:-}" ]]; then
+    # colored output mode
+    if [[ "${c}" = "${cur}" ]]; then
+      echo "${cur_ctx_bg}${cur_ctx_fg}${c}${normal}"
+    else
+      echo "${c}"
+    fi
+  else
+    echo "${c}"
+  fi
+  done
+}
+
+swap_namespace() {
+  local ctx ns
+  ctx="$(current_context)" || exit_err "error getting current context"
+  ns="$(read_namespace "${ctx}")"
+  if [[ -z "${ns}" ]]; then
+    echo "error: No previous namespace found for current context." >&2
+    exit 1
+  fi
+  set_namespace "${ns}"
+}
+
+main() {
+  if [[ -z "${KUBECTL:-}" ]]; then
+    if hash kubectl 2>/dev/null; then
+      KUBECTL=kubectl
+    elif hash kubectl.exe  2>/dev/null; then
+      KUBECTL=kubectl.exe
+    else
+      echo >&2 "kubectl is not installed"
+      exit 1
+    fi
+  fi
+
+  if [[ "$#" -eq 0 ]]; then
+    if [[ -t 1 && -z ${KUBECTX_IGNORE_FZF:-} && "$(type fzf &>/dev/null; echo $?)" -eq 0 ]]; then
+      choose_namespace_interactive
+    else
+      list_namespaces
+    fi
+  elif [[ "$#" -eq 1 ]]; then
+    if [[ "${1}" == '-h' || "${1}" == '--help' ]]; then
+      usage
+    elif [[ "${1}" == "-" ]]; then
+      swap_namespace
+    elif [[ "${1}" == '-c' || "${1}" == '--current' ]]; then
+      current_namespace
+    elif [[ "${1}" =~ ^-(.*) ]]; then
+      echo "error: unrecognized flag \"${1}\"" >&2
+      usage
+      exit 1
+    elif [[ "${1}" =~ (.+)=(.+) ]]; then
+      alias_context "${BASH_REMATCH[2]}" "${BASH_REMATCH[1]}"
+    else
+      set_namespace "${1}"
+    fi
+  else
+    echo "error: too many flags" >&2
+    usage
+    exit 1
+  fi
+}
+
+main "$@"

roles/k8s_manager/tasks/main.yml

@@ -77,12 +77,12 @@
     disable_gpg_check: true
 
 
-- name: k9s | Set permissions
+#- name: k9s | Set permissions
-  ansible.builtin.file:
+#  ansible.builtin.file:
-    path: /usr/local/bin/k9s
+#    path: /usr/local/bin/k9s
-    mode: "0755"
+#    mode: "0755"
-    owner: root
+#    owner: root
-    group: root
+#    group: root
 
 - name: kubectx/kubens | Download kubectx
   ansible.builtin.get_url:

roles/k8s_worker/tasks/prerequisites.yml

@@ -27,7 +27,7 @@
 
 - name: Prerequisites | Load kernel modules (persistent)
   ansible.builtin.copy:
-    dest: /etc/modules-load.d/k8s.conf
+    dest: /etc/modules-load.d/99-k8s.conf
     content: |
       overlay
       br_netfilter
@@ -45,7 +45,7 @@
   ansible.posix.sysctl:
     name: "{{ item.key }}"
     value: "{{ item.value }}"
-    sysctl_file: /etc/sysctl.d/k8s.conf
+    sysctl_file: /etc/sysctl.d/99-k8s.conf
     reload: true
   loop:
     - { key: "net.bridge.bridge-nf-call-iptables", value: "1" }

roles/longhorn/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-longhorn_chart_version: "1.7.2"
+longhorn_chart_version: "1.12.0"
 longhorn_namespace: longhorn-system
 
 # Number of replicas per volume. Set to 1 while only 1 worker node exists.

roles/longhorn/tasks/disks.yml

@@ -11,8 +11,8 @@
   loop: "{{ groups['workers'] }}"
   register: _lh_node_wait
   until: _lh_node_wait.rc == 0
-  retries: 24
+  retries: 40
-  delay: 5
+  delay: 15
   changed_when: false
   failed_when: _lh_node_wait.rc != 0 and _lh_node_wait.attempts | default(0) >= 24
   environment:

roles/traefik/defaults/main.yml

@@ -1,5 +1,5 @@
 ---
-traefik_chart_version: "32.1.0"
+traefik_chart_version: "41.0.2"
 traefik_namespace: traefik
 traefik_kubeconfig: "/home/{{ ansible_user }}/.kube/config"
 traefik_port_map: []

roles/traefik/templates/traefik-values.yml.j2

@@ -45,8 +45,10 @@ providers:
 persistence:
   enabled: false
 
-logs:
+log:
-  general:
+  level: ERROR
-    level: INFO
+  format: json
-  access:
+
+accessLog:
   enabled: true
+  format: json