Add Loader Cronjob for ElasticSearch
This commit is contained in:
parent
cad9ac8653
commit
161d6ee7c6
2
el-pipeline/app/kustomization.yaml
Normal file
2
el-pipeline/app/kustomization.yaml
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
resources:
|
||||||
|
- ../base
|
||||||
57
el-pipeline/base/elt-es.cronjob.yaml
Normal file
57
el-pipeline/base/elt-es.cronjob.yaml
Normal file
@ -0,0 +1,57 @@
|
|||||||
|
apiVersion: batch/v1
|
||||||
|
kind: CronJob
|
||||||
|
metadata:
|
||||||
|
name: elt-es
|
||||||
|
namespace: transcity-demo-aws-pg
|
||||||
|
spec:
|
||||||
|
schedule: "5 * * * *"
|
||||||
|
concurrencyPolicy: Forbid
|
||||||
|
suspend: false
|
||||||
|
jobTemplate:
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
restartPolicy: Never
|
||||||
|
nodeSelector:
|
||||||
|
workload: transcity-pg
|
||||||
|
containers:
|
||||||
|
- name: es-elt
|
||||||
|
image: 767397786784.dkr.ecr.eu-west-3.amazonaws.com/reports/es-elt:0.1.4
|
||||||
|
# args:
|
||||||
|
# - --start-from
|
||||||
|
# - 2026-01-26/00
|
||||||
|
env:
|
||||||
|
- name: ELT_ELASTICSEARCH_URL
|
||||||
|
value: http://elasticsearch-master.reports-sds.svc:9200
|
||||||
|
- name: ELT_ELASTICSEARCH_USER
|
||||||
|
value: elastic
|
||||||
|
- name: ELT_ELASTICSEARCH_PASSWORD
|
||||||
|
value: GTRNZHCJTiGJ2CfY
|
||||||
|
- name: ELT_MAX_WORKERS
|
||||||
|
value: "2"
|
||||||
|
volumeMounts:
|
||||||
|
- name: export
|
||||||
|
mountPath: /data
|
||||||
|
- name: state
|
||||||
|
mountPath: /state
|
||||||
|
volumes:
|
||||||
|
- name: export
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: demo-aws-pg-dih-export
|
||||||
|
- name: state
|
||||||
|
persistentVolumeClaim:
|
||||||
|
claimName: elt-es
|
||||||
|
parallelism: 1
|
||||||
|
backoffLimit: 3
|
||||||
|
---
|
||||||
|
apiVersion: v1
|
||||||
|
kind: PersistentVolumeClaim
|
||||||
|
metadata:
|
||||||
|
name: elt-es
|
||||||
|
namespace: transcity-demo-aws-pg
|
||||||
|
spec:
|
||||||
|
accessModes:
|
||||||
|
- ReadWriteOnce
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 16Mi
|
||||||
2
el-pipeline/base/kustomization.yaml
Normal file
2
el-pipeline/base/kustomization.yaml
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
resources:
|
||||||
|
- elt-es.cronjob.yaml
|
||||||
2
el-pipeline/local/kustomization.yaml
Normal file
2
el-pipeline/local/kustomization.yaml
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
resources:
|
||||||
|
- ../app
|
||||||
Loading…
x
Reference in New Issue
Block a user