From b32ea3034577f377d19cb781ad2de696c66c8f53 Mon Sep 17 00:00:00 2001 From: mrwho Date: Thu, 16 Jul 2026 21:28:44 +0300 Subject: [PATCH] refactor(infra): restructure inventory and update dependencies - rename manager node and update host FQDNs in inventory - enable multiple worker nodes by commenting out secondary disk - standardize k8s configuration file paths with numeric prefixes - disable k9s permissions step in manager role - upgrade Longhorn chart to v1.12.0 and Traefik chart to v41.0.2 - update Traefik port_map with protocol and add forgejo service - refine Traefik logging configuration to use new JSON format - increase Longhorn disk probe retries and delays - add placeholder directory for k8s_manager files --- inventory/prod/group_vars/traefik.yml | 24 +- inventory/prod/group_vars/workers.yml | 4 +- inventory/prod/hosts.yml | 12 +- .../k8s_control_plane/tasks/prerequisites.yml | 4 +- roles/k8s_manager/files/kubectx | 254 ++++++++++++++++++ roles/k8s_manager/files/kubens | 230 ++++++++++++++++ roles/k8s_manager/tasks/main.yml | 12 +- roles/k8s_worker/tasks/prerequisites.yml | 4 +- roles/longhorn/defaults/main.yml | 2 +- roles/longhorn/tasks/disks.yml | 4 +- roles/traefik/defaults/main.yml | 2 +- roles/traefik/templates/traefik-values.yml.j2 | 12 +- 12 files changed, 532 insertions(+), 32 deletions(-) create mode 100755 roles/k8s_manager/files/kubectx create mode 100755 roles/k8s_manager/files/kubens diff --git a/inventory/prod/group_vars/traefik.yml b/inventory/prod/group_vars/traefik.yml index f5897b5..8da9c2f 100644 --- a/inventory/prod/group_vars/traefik.yml +++ b/inventory/prod/group_vars/traefik.yml @@ -1,31 +1,45 @@ --- -traefik_enable_ingress: true -traefik_ingress_class_name: "traefik" -traefik_namespace: kube-system +#traefik_enable_ingress: true +#traefik_ingress_class_name: "traefik" +#traefik_namespace: kube-system traefik_port_map: - name: k8s-dashboard port: 10001 + protocol: tcp backend: namespace: kubernetes-dashboard service: kubernetes-dashboard port: 443 - scheme: https +# scheme: https basicauth: enabled: false - name: longhorn-ui port: 10002 + protocol: tcp backend: namespace: longhorn-system service: longhorn-frontend port: 80 - scheme: http +# scheme: http basicauth: enabled: true secret_name: traefik-auth-longhorn + - name: forgejo + port: 10003 + protocol: tcp + backend: + namespace: git + service: forgejo-http + port: 3000 +# scheme: http + basicauth: + enabled: false + + # Template for future services: # - name: my-service # description: "Human-readable description" diff --git a/inventory/prod/group_vars/workers.yml b/inventory/prod/group_vars/workers.yml index 3fb3258..db00a0d 100644 --- a/inventory/prod/group_vars/workers.yml +++ b/inventory/prod/group_vars/workers.yml @@ -8,5 +8,5 @@ longhorn_disks: - device: /dev/sdb mountpoint: /mnt/longhorn-disk1 - - device: /dev/sdc - mountpoint: /mnt/longhorn-disk2 +# - device: /dev/sdc +# mountpoint: /mnt/longhorn-disk2 diff --git a/inventory/prod/hosts.yml b/inventory/prod/hosts.yml index f9577fe..010b4a5 100644 --- a/inventory/prod/hosts.yml +++ b/inventory/prod/hosts.yml @@ -4,20 +4,20 @@ all: # Operator workstation: kubectl, helm, k9s manager_nodes: hosts: - k8s-manager-01: - ansible_host: 172.20.21.200 + mgmt.local.mrwho.ru: + ansible_host: 172.20.21.199 # Kubernetes cluster nodes k8s_cluster: children: control_plane: hosts: - k8s-master-01: - ansible_host: 172.20.21.201 + k8s-master-01.local.mrwho.ru: + ansible_host: 172.20.21.200 workers: hosts: - k8s-worker-01: - ansible_host: 172.20.21.202 + k8s-worker-01.local.mrwho.ru: + ansible_host: 172.20.21.201 longhorn_disks: - device: /dev/sdb mountpoint: /mnt/longhorn-disk1 diff --git a/roles/k8s_control_plane/tasks/prerequisites.yml b/roles/k8s_control_plane/tasks/prerequisites.yml index d3d10d0..b462a24 100644 --- a/roles/k8s_control_plane/tasks/prerequisites.yml +++ b/roles/k8s_control_plane/tasks/prerequisites.yml @@ -27,7 +27,7 @@ - name: Prerequisites | Load kernel modules (persistent) ansible.builtin.copy: - dest: /etc/modules-load.d/k8s.conf + dest: /etc/modules-load.d/99-k8s.conf content: | overlay br_netfilter @@ -45,7 +45,7 @@ ansible.posix.sysctl: name: "{{ item.key }}" value: "{{ item.value }}" - sysctl_file: /etc/sysctl.d/k8s.conf + sysctl_file: /etc/sysctl.d/99-k8s.conf reload: true loop: - { key: "net.bridge.bridge-nf-call-iptables", value: "1" } diff --git a/roles/k8s_manager/files/kubectx b/roles/k8s_manager/files/kubectx new file mode 100755 index 0000000..f00bbf3 --- /dev/null +++ b/roles/k8s_manager/files/kubectx @@ -0,0 +1,254 @@ +#!/usr/bin/env bash +# +# kubectx(1) is a utility to manage and switch between kubectl contexts. + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[[ -n $DEBUG ]] && set -x + +set -eou pipefail +IFS=$'\n\t' + +SELF_CMD="$0" + +KUBECTX="${XDG_CACHE_HOME:-$HOME/.kube}/kubectx" + +usage() { + local SELF + SELF="kubectx" + if [[ "$(basename "$0")" == kubectl-* ]]; then # invoked as plugin + SELF="kubectl ctx" + fi + + cat < : switch to context + $SELF - : switch to the previous context + $SELF -c, --current : show the current context name + $SELF = : rename context to + $SELF =. : rename current-context to + $SELF -d [] : delete context ('.' for current-context) + (this command won't delete the user/cluster entry + that is used by the context) + $SELF -u, --unset : unset the current context + + $SELF -h,--help : show this message + + (This executable is the legacy bash-based implementation, consider upgrading to Go-based implementation.) +EOF +} + +exit_err() { + echo >&2 "${1}" + exit 1 +} + +current_context() { + $KUBECTL config view -o=jsonpath='{.current-context}' +} + +get_contexts() { + $KUBECTL config get-contexts -o=name | sort -n +} + +list_contexts() { + set -u pipefail + local cur ctx_list + cur="$(current_context)" || exit_err "error getting current context" + ctx_list=$(get_contexts) || exit_err "error getting context list" + + local yellow darkbg normal + yellow=$(tput setaf 3 || true) + darkbg=$(tput setab 0 || true) + normal=$(tput sgr0 || true) + + local cur_ctx_fg cur_ctx_bg + cur_ctx_fg=${KUBECTX_CURRENT_FGCOLOR:-$yellow} + cur_ctx_bg=${KUBECTX_CURRENT_BGCOLOR:-$darkbg} + + for c in $ctx_list; do + if [[ -n "${_KUBECTX_FORCE_COLOR:-}" || \ + -t 1 && -z "${NO_COLOR:-}" ]]; then + # colored output mode + if [[ "${c}" = "${cur}" ]]; then + echo "${cur_ctx_bg}${cur_ctx_fg}${c}${normal}" + else + echo "${c}" + fi + else + echo "${c}" + fi + done +} + +read_context() { + if [[ -f "${KUBECTX}" ]]; then + cat "${KUBECTX}" + fi +} + +save_context() { + local saved + saved="$(read_context)" + + if [[ "${saved}" != "${1}" ]]; then + printf %s "${1}" > "${KUBECTX}" + fi +} + +switch_context() { + $KUBECTL config use-context "${1}" +} + +choose_context_interactive() { + local choice + choice="$(_KUBECTX_FORCE_COLOR=1 \ + FZF_DEFAULT_COMMAND="${SELF_CMD}" \ + fzf --ansi --no-preview || true)" + if [[ -z "${choice}" ]]; then + echo 2>&1 "error: you did not choose any of the options" + exit 1 + else + set_context "${choice}" + fi +} + +set_context() { + local prev + prev="$(current_context)" || exit_err "error getting current context" + + switch_context "${1}" + + if [[ "${prev}" != "${1}" ]]; then + save_context "${prev}" + fi +} + +swap_context() { + local ctx + ctx="$(read_context)" + if [[ -z "${ctx}" ]]; then + echo "error: No previous context found." >&2 + exit 1 + fi + set_context "${ctx}" +} + +context_exists() { + grep -q ^"${1}"\$ <($KUBECTL config get-contexts -o=name) +} + +rename_context() { + local old_name="${1}" + local new_name="${2}" + + if [[ "${old_name}" == "." ]]; then + old_name="$(current_context)" + fi + + if ! context_exists "${old_name}"; then + echo "error: Context \"${old_name}\" not found, can't rename it." >&2 + exit 1 + fi + + if context_exists "${new_name}"; then + echo "Context \"${new_name}\" exists, deleting..." >&2 + $KUBECTL config delete-context "${new_name}" 1>/dev/null 2>&1 + fi + + $KUBECTL config rename-context "${old_name}" "${new_name}" +} + +delete_contexts() { + for i in "${@}"; do + delete_context "${i}" + done +} + +delete_context() { + local ctx + ctx="${1}" + if [[ "${ctx}" == "." ]]; then + ctx="$(current_context)" || exit_err "error getting current context" + fi + echo "Deleting context \"${ctx}\"..." >&2 + $KUBECTL config delete-context "${ctx}" +} + +unset_context() { + echo "Unsetting current context." >&2 + $KUBECTL config unset current-context +} + +main() { + if [[ -z "${KUBECTL:-}" ]]; then + if hash kubectl 2>/dev/null; then + KUBECTL=kubectl + elif hash kubectl.exe 2>/dev/null; then + KUBECTL=kubectl.exe + else + echo >&2 "kubectl is not installed" + exit 1 + fi + fi + + if [[ "$#" -eq 0 ]]; then + if [[ -t 1 && -z "${KUBECTX_IGNORE_FZF:-}" && "$(type fzf &>/dev/null; echo $?)" -eq 0 ]]; then + choose_context_interactive + else + list_contexts + fi + elif [[ "${1}" == "-d" ]]; then + if [[ "$#" -lt 2 ]]; then + echo "error: missing context NAME" >&2 + usage + exit 1 + fi + delete_contexts "${@:2}" + elif [[ "$#" -gt 1 ]]; then + echo "error: too many arguments" >&2 + usage + exit 1 + elif [[ "$#" -eq 1 ]]; then + if [[ "${1}" == "-" ]]; then + swap_context + elif [[ "${1}" == '-c' || "${1}" == '--current' ]]; then + # we don't call current_context here for two reasons: + # - it does not fail when current-context property is not set + # - it does not return a trailing newline + $KUBECTL config current-context + elif [[ "${1}" == '-u' || "${1}" == '--unset' ]]; then + unset_context + elif [[ "${1}" == '-h' || "${1}" == '--help' ]]; then + usage + elif [[ "${1}" =~ ^-(.*) ]]; then + echo "error: unrecognized flag \"${1}\"" >&2 + usage + exit 1 + elif [[ "${1}" =~ (.+)=(.+) ]]; then + rename_context "${BASH_REMATCH[2]}" "${BASH_REMATCH[1]}" + else + set_context "${1}" + fi + else + usage + exit 1 + fi +} + +main "$@" diff --git a/roles/k8s_manager/files/kubens b/roles/k8s_manager/files/kubens new file mode 100755 index 0000000..1088915 --- /dev/null +++ b/roles/k8s_manager/files/kubens @@ -0,0 +1,230 @@ +#!/usr/bin/env bash +# +# kubens(1) is a utility to switch between Kubernetes namespaces. + +# Copyright 2017 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[[ -n $DEBUG ]] && set -x + +set -eou pipefail +IFS=$'\n\t' + +SELF_CMD="$0" + +KUBENS_DIR="${XDG_CACHE_HOME:-$HOME/.kube}/kubens" + +usage() { + local SELF + SELF="kubens" + if [[ "$(basename "$0")" == kubectl-* ]]; then # invoked as plugin + SELF="kubectl ns" + fi + + cat < : change the active namespace of current context + $SELF - : switch to the previous namespace in this context + $SELF -c, --current : show the current namespace + $SELF -h,--help : show this message + + (This executable is the legacy bash-based implementation, consider upgrading to Go-based implementation.) +EOF +} + +exit_err() { + echo >&2 "${1}" + exit 1 +} + +current_namespace() { + local cur_ctx + + cur_ctx="$(current_context)" || exit_err "error getting current context" + ns="$($KUBECTL config view -o=jsonpath="{.contexts[?(@.name==\"${cur_ctx}\")].context.namespace}")" \ + || exit_err "error getting current namespace" + + if [[ -z "${ns}" ]]; then + echo "default" + else + echo "${ns}" + fi +} + +current_context() { + $KUBECTL config current-context +} + +get_namespaces() { + $KUBECTL get namespaces -o=jsonpath='{range .items[*].metadata.name}{@}{"\n"}{end}' +} + +escape_context_name() { + echo "${1//\//-}" +} + +namespace_file() { + local ctx + + ctx="$(escape_context_name "${1}")" + echo "${KUBENS_DIR}/${ctx}" +} + +read_namespace() { + local f + f="$(namespace_file "${1}")" + [[ -f "${f}" ]] && cat "${f}" + return 0 +} + +save_namespace() { + mkdir -p "${KUBENS_DIR}" + local f saved + f="$(namespace_file "${1}")" + saved="$(read_namespace "${1}")" + + if [[ "${saved}" != "${2}" ]]; then + printf %s "${2}" > "${f}" + fi +} + +switch_namespace() { + local ctx="${1}" + $KUBECTL config set-context "${ctx}" --namespace="${2}" + echo "Active namespace is \"${2}\".">&2 +} + +choose_namespace_interactive() { + # directly calling kubens via fzf might fail with a cryptic error like + # "$FZF_DEFAULT_COMMAND failed", so try to see if we can list namespaces + # locally first + if [[ -z "$(list_namespaces)" ]]; then + echo >&2 "error: could not list namespaces (is the cluster accessible?)" + exit 1 + fi + + local choice + choice="$(_KUBECTX_FORCE_COLOR=1 \ + FZF_DEFAULT_COMMAND="${SELF_CMD}" \ + fzf --ansi --no-preview || true)" + if [[ -z "${choice}" ]]; then + echo 2>&1 "error: you did not choose any of the options" + exit 1 + else + set_namespace "${choice}" + fi +} + +set_namespace() { + local ctx prev + ctx="$(current_context)" || exit_err "error getting current context" + prev="$(current_namespace)" || exit_error "error getting current namespace" + + if grep -q ^"${1}"\$ <(get_namespaces); then + switch_namespace "${ctx}" "${1}" + + if [[ "${prev}" != "${1}" ]]; then + save_namespace "${ctx}" "${prev}" + fi + else + echo "error: no namespace exists with name \"${1}\".">&2 + exit 1 + fi +} + +list_namespaces() { + local yellow darkbg normal + yellow=$(tput setaf 3 || true) + darkbg=$(tput setab 0 || true) + normal=$(tput sgr0 || true) + + local cur_ctx_fg cur_ctx_bg + cur_ctx_fg=${KUBECTX_CURRENT_FGCOLOR:-$yellow} + cur_ctx_bg=${KUBECTX_CURRENT_BGCOLOR:-$darkbg} + + local cur ns_list + cur="$(current_namespace)" || exit_err "error getting current namespace" + ns_list=$(get_namespaces) || exit_err "error getting namespace list" + + for c in $ns_list; do + if [[ -n "${_KUBECTX_FORCE_COLOR:-}" || \ + -t 1 && -z "${NO_COLOR:-}" ]]; then + # colored output mode + if [[ "${c}" = "${cur}" ]]; then + echo "${cur_ctx_bg}${cur_ctx_fg}${c}${normal}" + else + echo "${c}" + fi + else + echo "${c}" + fi + done +} + +swap_namespace() { + local ctx ns + ctx="$(current_context)" || exit_err "error getting current context" + ns="$(read_namespace "${ctx}")" + if [[ -z "${ns}" ]]; then + echo "error: No previous namespace found for current context." >&2 + exit 1 + fi + set_namespace "${ns}" +} + +main() { + if [[ -z "${KUBECTL:-}" ]]; then + if hash kubectl 2>/dev/null; then + KUBECTL=kubectl + elif hash kubectl.exe 2>/dev/null; then + KUBECTL=kubectl.exe + else + echo >&2 "kubectl is not installed" + exit 1 + fi + fi + + if [[ "$#" -eq 0 ]]; then + if [[ -t 1 && -z ${KUBECTX_IGNORE_FZF:-} && "$(type fzf &>/dev/null; echo $?)" -eq 0 ]]; then + choose_namespace_interactive + else + list_namespaces + fi + elif [[ "$#" -eq 1 ]]; then + if [[ "${1}" == '-h' || "${1}" == '--help' ]]; then + usage + elif [[ "${1}" == "-" ]]; then + swap_namespace + elif [[ "${1}" == '-c' || "${1}" == '--current' ]]; then + current_namespace + elif [[ "${1}" =~ ^-(.*) ]]; then + echo "error: unrecognized flag \"${1}\"" >&2 + usage + exit 1 + elif [[ "${1}" =~ (.+)=(.+) ]]; then + alias_context "${BASH_REMATCH[2]}" "${BASH_REMATCH[1]}" + else + set_namespace "${1}" + fi + else + echo "error: too many flags" >&2 + usage + exit 1 + fi +} + +main "$@" diff --git a/roles/k8s_manager/tasks/main.yml b/roles/k8s_manager/tasks/main.yml index 85e8ff2..2d41f5d 100644 --- a/roles/k8s_manager/tasks/main.yml +++ b/roles/k8s_manager/tasks/main.yml @@ -77,12 +77,12 @@ disable_gpg_check: true -- name: k9s | Set permissions - ansible.builtin.file: - path: /usr/local/bin/k9s - mode: "0755" - owner: root - group: root +#- name: k9s | Set permissions +# ansible.builtin.file: +# path: /usr/local/bin/k9s +# mode: "0755" +# owner: root +# group: root - name: kubectx/kubens | Download kubectx ansible.builtin.get_url: diff --git a/roles/k8s_worker/tasks/prerequisites.yml b/roles/k8s_worker/tasks/prerequisites.yml index e5cc54d..842fea9 100644 --- a/roles/k8s_worker/tasks/prerequisites.yml +++ b/roles/k8s_worker/tasks/prerequisites.yml @@ -27,7 +27,7 @@ - name: Prerequisites | Load kernel modules (persistent) ansible.builtin.copy: - dest: /etc/modules-load.d/k8s.conf + dest: /etc/modules-load.d/99-k8s.conf content: | overlay br_netfilter @@ -45,7 +45,7 @@ ansible.posix.sysctl: name: "{{ item.key }}" value: "{{ item.value }}" - sysctl_file: /etc/sysctl.d/k8s.conf + sysctl_file: /etc/sysctl.d/99-k8s.conf reload: true loop: - { key: "net.bridge.bridge-nf-call-iptables", value: "1" } diff --git a/roles/longhorn/defaults/main.yml b/roles/longhorn/defaults/main.yml index 4710569..e802fda 100644 --- a/roles/longhorn/defaults/main.yml +++ b/roles/longhorn/defaults/main.yml @@ -1,5 +1,5 @@ --- -longhorn_chart_version: "1.7.2" +longhorn_chart_version: "1.12.0" longhorn_namespace: longhorn-system # Number of replicas per volume. Set to 1 while only 1 worker node exists. diff --git a/roles/longhorn/tasks/disks.yml b/roles/longhorn/tasks/disks.yml index b71f033..ada4f03 100644 --- a/roles/longhorn/tasks/disks.yml +++ b/roles/longhorn/tasks/disks.yml @@ -11,8 +11,8 @@ loop: "{{ groups['workers'] }}" register: _lh_node_wait until: _lh_node_wait.rc == 0 - retries: 24 - delay: 5 + retries: 40 + delay: 15 changed_when: false failed_when: _lh_node_wait.rc != 0 and _lh_node_wait.attempts | default(0) >= 24 environment: diff --git a/roles/traefik/defaults/main.yml b/roles/traefik/defaults/main.yml index 74e04d5..c5d4dcf 100644 --- a/roles/traefik/defaults/main.yml +++ b/roles/traefik/defaults/main.yml @@ -1,5 +1,5 @@ --- -traefik_chart_version: "32.1.0" +traefik_chart_version: "41.0.2" traefik_namespace: traefik traefik_kubeconfig: "/home/{{ ansible_user }}/.kube/config" traefik_port_map: [] diff --git a/roles/traefik/templates/traefik-values.yml.j2 b/roles/traefik/templates/traefik-values.yml.j2 index 5708b54..aa50b11 100644 --- a/roles/traefik/templates/traefik-values.yml.j2 +++ b/roles/traefik/templates/traefik-values.yml.j2 @@ -45,8 +45,10 @@ providers: persistence: enabled: false -logs: - general: - level: INFO - access: - enabled: true +log: + level: ERROR + format: json + +accessLog: + enabled: true + format: json