Can we combine jobs into one in .gitlab ci/cd yaml file?

Question

This is ci/cd yaml file I using

 services:
      - docker:19.03.11-dind
    workflow:
      rules:
        - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "developer" || $CI_COMMIT_BRANCH == "stage"|| ($CI_COMMIT_BRANCH =~ (/^([A-Z]([0-9][-_])?)?SPRINT(([-_][A-Z][0-9])?)+/i))
          when: always
        - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH != "developer" || $CI_COMMIT_BRANCH != "stage"|| ($CI_COMMIT_BRANCH !~ (/^([A-Z]([0-9][-_])?)?SPRINT(([-_][A-Z][0-9])?)+/i))
          when: never 
    stages:
      - build
      - Publish
      - deploy
    cache:
      paths:
        - .m2/repository
        - target
    build_jar:
      image: maven:3.8.3-jdk-11
      stage: build
      script: 
        - mvn clean install package -DskipTests=true
      artifacts:
        paths:
          - target/*.jar
    
    docker_build_dev:
      stage: Publish
      image: docker:19.03.11
      services:
        - docker:19.03.11-dind      
      variables:
        IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
      script: 
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        - docker build --build-arg environment_name= development  -t $IMAGE_TAG .
        - docker push $IMAGE_TAG
      only:
        - developer 
    
    docker_build_stage:
      stage: Publish
      image: docker:19.03.11
      services:
        - docker:19.03.11-dind   
      variables:
        IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
      script: 
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        - docker build --build-arg environment_name= stage  -t $IMAGE_TAG .
        - docker push $IMAGE_TAG   
      only:
        - stage    
         
    deploy_dev:
      stage: deploy
      image: stellacenter/aws-helm-kubectl
      before_script:
        - aws configure set aws_access_key_id ${DEV_AWS_ACCESS_KEY_ID}
        - aws configure set aws_secret_access_key ${DEV_AWS_SECRET_ACCESS_KEY}
        - aws configure set region ${DEV_AWS_DEFAULT_REGION}
      script:
        - sed -i "s/<VERSION>/${CI_COMMIT_SHORT_SHA}/g" provider-service.yml     
        - mkdir -p  $HOME/.kube
        - cp $KUBE_CONFIG_DEV $HOME/.kube/config
        - chown $(id -u):$(id -g) $HOME/.kube/config 
        - export KUBECONFIG=$HOME/.kube/config
        - kubectl apply -f  provider-service.yml -n ${KUBE_NAMESPACE_DEV}
      only:
        - developer  
    
    deploy_stage:
      stage: deploy
      image: stellacenter/aws-helm-kubectl
      before_script:
        - aws configure set aws_access_key_id ${DEV_AWS_ACCESS_KEY_ID}
        - aws configure set aws_secret_access_key ${DEV_AWS_SECRET_ACCESS_KEY}
        - aws configure set region ${DEV_AWS_DEFAULT_REGION}
      script:
        - sed -i "s/<VERSION>/${CI_COMMIT_SHORT_SHA}/g" provider-service.yml    
        - mkdir -p  $HOME/.kube
        - cp $KUBE_CONFIG_STAGE $HOME/.kube/config
        - chown $(id -u):$(id -g) $HOME/.kube/config 
        - export KUBECONFIG=$HOME/.kube/config
        - kubectl apply -f  provider-service.yml -n ${KUBE_NAMESPACE_STAGE}
      only:
        - stage 

But currently I want to combine the stages of publish & deploy? I done but it shows some error in publish stage

services:
  - docker:19.03.11-dind
workflow:
  rules:
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "developer" || $CI_COMMIT_BRANCH == "stage"|| ($CI_COMMIT_BRANCH =~ (/^([A-Z]([0-9][-_])?)?SPRINT(([-_][A-Z][0-9])?)+/i))
      when: always
    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH != "developer" || $CI_COMMIT_BRANCH != "stage"|| ($CI_COMMIT_BRANCH !~ (/^([A-Z]([0-9][-_])?)?SPRINT(([-_][A-Z][0-9])?)+/i))
      when: never 
stages:
  - build
  - Publish
  - deploy
cache:
  paths:
    - .m2/repository
    - target
    
    build_jar:
      image: maven:3.8.3-jdk-11
      stage: build
      script: 
        - mvn clean install package -DskipTests=true
      artifacts:
        paths:
          - target/*.jar
    
    docker_build:
      stage: Publish
      image: docker:19.03.11
      services:
        - docker:19.03.11-dind      
      variables:
        IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHORT_SHA
      script: 
        - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
        - docker build --build-arg environment_name= development  -t $IMAGE_TAG .
        - docker build --build-arg environment_name= stage  -t $IMAGE_TAG .
        - docker push $IMAGE_TAG
      only:
        - developer
        - stage    
         
    deploy_job:
      stage: deploy
      image: stellacenter/aws-helm-kubectl
      before_script:
        - aws configure set aws_access_key_id ${DEV_AWS_ACCESS_KEY_ID}
        - aws configure set aws_secret_access_key ${DEV_AWS_SECRET_ACCESS_KEY}
        - aws configure set region ${DEV_AWS_DEFAULT_REGION}
      script:
        - sed -i "s/<VERSION>/${CI_COMMIT_SHORT_SHA}/g" provider-service.yml     
        - mkdir -p  $HOME/.kube
        - cp $KUBE_CONFIG_DEV $HOME/.kube/config
        - chown $(id -u):$(id -g) $HOME/.kube/config 
        - export KUBECONFIG=$HOME/.kube/config
        - kubectl apply -f  provider-service.yml -n ${KUBE_NAMESPACE_DEV}
        - kubectl apply -f  provider-service.yml -n ${KUBE_NAMESPACE_STAGE}
      only:
        - developer
        - stage 

This is the one , I used now but it shows error

$ docker build --build-arg environment_name= development  -t $IMAGE_TAG .
"docker build" requires exactly 1 argument.
See 'docker build --help'.
Usage:  docker build [OPTIONS] PATH | URL | -
Build an image from a Dockerfile
Cleaning up project directory and file based variables
00:01
ERROR: Job failed: exit code 1

My problem is , I'm combining two branch (stage & developer) yaml scripts and files , for the single line like "--build-arg environment_name=development" for developer "--build-arg environment_name=stage" for stage likely I separating the jobs for this single line, so only I'm asking that , is there any possibility for combining the script? So I enclosed the full script which is divided and also combined one @Bichon Motive: want to combine the two publish(developer and stage) and deploy (developer and stage)jobs into single job

Answer 1

Following my comments, here is my understanding of the problem, what I think is wrong in the solution attempt, my solution and finally the limitations of what you want to achieve.

Problem

An original Gitlab-CI script build an image docker and deploy associated Kubernetes resources for two environments: stage and development. These are respectively built and deployed by different jobs, each one executed for a dedicated branch (respectively stage and developer). Now, I guess, the two environments are merged into the same cluster. For some unknown reason, the question ask that pushing on either one of the two branches should build and deploy the two environments with the same code (which is almost sure to bring problems in the future if the service is not stateless but let's suppose it is). If this is not the problem to solve, let me know please.

Errors in your solution attempt

• as mentioned in comment, the docker error is raised because of the space in the --build-arg environment_name= development which should be --build-arg environment_name=development
• when you build your docker image, you give them a build argument that determines for which environment they are built. However, you are using the same docker tag ${IMAGE_TAG} for both docker images. So what happens is that your last built image (the one for stage) will be deployed for both environments which is not what you want.

Solution

We fix the docker error and use different docker image names for each environment. Also, we must sed the version for both environment before deployment so I created temporary copies of the original deployment.

docker_build:
  stage: Publish
  image: docker:19.03.11
  services:
    - docker:19.03.11-dind      
  variables:
    IMAGE_TAG_DEV: $CI_REGISTRY_IMAGE:dev-$CI_COMMIT_SHORT_SHA
    IMAGE_TAG_STAGE: $CI_REGISTRY_IMAGE:stage-$CI_COMMIT_SHORT_SHA
  script: 
    - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
    - docker build --build-arg environment_name=development -t "${IMAGE_TAG_DEV}" .
    - docker build --build-arg environment_name=stage -t "${IMAGE_TAG_STAGE}" .
    - docker push "${IMAGE_TAG_DEV}" "${IMAGE_TAG_STAGE}"
  only:
    - developer
    - stage    
     
deploy_job:
  stage: deploy
  image: stellacenter/aws-helm-kubectl
  before_script:
    - aws configure set aws_access_key_id ${DEV_AWS_ACCESS_KEY_ID}
    - aws configure set aws_secret_access_key ${DEV_AWS_SECRET_ACCESS_KEY}
    - aws configure set region ${DEV_AWS_DEFAULT_REGION}
  script:
    - cp provider-service.yml provider-service-dev.yml
    - cp provider-service.yml provider-service-stage.yml
    - sed -i "s/<VERSION>/dev-${CI_COMMIT_SHORT_SHA}/g" provider-service-dev.yml
    - sed -i "s/<VERSION>/stage-${CI_COMMIT_SHORT_SHA}/g" provider-service-stage.yml
    - mkdir -p  $HOME/.kube
    - cp $KUBE_CONFIG_DEV $HOME/.kube/config
    - chown $(id -u):$(id -g) $HOME/.kube/config 
    - export KUBECONFIG=$HOME/.kube/config
    - kubectl apply -f provider-service-dev.yml -n ${KUBE_NAMESPACE_DEV}
    - kubectl apply -f provider-service-stage.yml -n ${KUBE_NAMESPACE_STAGE}
  only:
    - developer
    - stage

Limitations

Beware that such a workflow (having two distinct branches deploying, potentially in concurrence, two environments at the same time) is not really recommended and can lead to versioning problems, especially if your service is not stateless but also if the branches diverge. So before using this, I would advise to ensure that your service is stateless and also, for instance, that merge requests are fast-forward only.