Azure container instance still creating when using docker compose and resources limits

Bruno MC 1 Reputation point
2022-11-07T09:28:54.373+00:00

Hello,

I am trying to deploy my containerized application to azure using docker compose aci context. For this I am following this tutorial: tutorial-docker-compose

It works fine, when I use a simple docker compose definition with only 3 services. See below the docker-compose.yaml that works:

version: "3"  
  
services:  
  node1:  
    build:  
      context: server  
    image: ***.azurecr.io/node-server  
    command: ./server -config config/cloud-neighbour-1.yaml -socket-address 0.0.0.0 -socket-port 8881  
    ports:  
      - 8881  
    networks:  
      local:  
        ipv4_address: 172.28.1.101  
    deploy:  
      resources:  
        limits:  
          cpus: '0.50'  
          memory: 1G  
        reservations:  
          cpus: '0.50'  
          memory: 1G  
  node2:  
    build:  
      context: server  
    image: ***.azurecr.io/node-server  
    command: ./server -config config/cloud-neighbour-2.yaml -socket-address 0.0.0.0 -socket-port 8882  
    ports:  
      - 8882  
    networks:  
      local:  
        ipv4_address: 172.28.1.102  
    deploy:  
      resources:  
        limits:  
          cpus: '0.50'  
          memory: 1G  
        reservations:  
          cpus: '0.50'  
          memory: 1G  
  node3:  
    build:  
      context: server  
    image: ***.azurecr.io/node-server  
    command: ./server -config config/cloud-neighbour-3.yaml -socket-address 0.0.0.0 -socket-port 8883  
    ports:  
      - 8883  
    networks:  
      local:  
        ipv4_address: 172.28.1.103  
    deploy:  
      resources:  
        limits:  
          cpus: '0.50'  
          memory: 1G  
        reservations:  
          cpus: '0.50'  
          memory: 1G  
  
networks:  
  local:  
    driver: bridge  
    ipam:  
     config:  
       - subnet: 172.28.1.0/24  
         gateway: 172.28.1.1  

The problem is when I try to increase the number of services, the containers still creating and I have the following error.

$ docker compose --verbose -f .\nodes-123.docker-compose.yaml up  
level=debug msg="Up on project with name \"lab01-blockchain\""  
[+] Running 1/7  
 - Group lab01-blockchain  Created                                                                                                                                                    7.7s   
 - node1                   Creating                                                                                                                                                 900.0s  
 - node2                   Creating                                                                                                                                                 900.0s   
 - node3                   Creating                                                                                                                                                 900.0s   
 - node4                   Creating                                                                                                                                                 900.0s   
 - node5                   Creating                                                                                                                                                 900.0s   
 - node6                   Creating                                                                                                                                                 900.0s   
Future#WaitForCompletion: context has been cancelled: StatusCode=200 -- Original Error: context deadline exceeded  

The docker-compose.yaml with 6 services is the following:

version: "3"  
  
services:  
  node1:  
    # build:  
    #   context: server  
    image: ***.azurecr.io/node-server  
    command: ./server -config config/cloud-neighbour-1.yaml -socket-address 0.0.0.0 -socket-port 8881  
    ports:  
      - 8881  
    # networks:  
    #   local:  
    #     ipv4_address: 172.28.1.101  
    deploy:  
      resources:  
        limits:  
          cpus: '0.50'  
          memory: 1G  
        reservations:  
          cpus: '0.50'  
          memory: 1G  
  node2:  
    # build:  
    #   context: server  
    image: ***.azurecr.io/node-server  
    command: ./server -config config/cloud-neighbour-2.yaml -socket-address 0.0.0.0 -socket-port 8882  
    ports:  
      - 8882  
    # networks:  
    #   local:  
    #     ipv4_address: 172.28.1.102  
    deploy:  
      resources:  
        limits:  
          cpus: '0.50'  
          memory: 1G  
        reservations:  
          cpus: '0.50'  
          memory: 1G  
  node3:  
    # build:  
    #   context: server  
    image: ***.azurecr.io/node-server  
    command: ./server -config config/cloud-neighbour-3.yaml -socket-address 0.0.0.0 -socket-port 8883  
    ports:  
      - 8883  
    # networks:  
    #   local:  
    #     ipv4_address: 172.28.1.103  
    deploy:  
      resources:  
        limits:  
          cpus: '0.50'  
          memory: 1G  
        reservations:  
          cpus: '0.50'  
          memory: 1G  
  node4:  
    # build:  
    #   context: server  
    image: ***.azurecr.io/node-server  
    command: ./server -config config/cloud-neighbour-3.yaml -socket-address 0.0.0.0 -socket-port 8884  
    ports:  
      - 8884  
    # networks:  
    #   local:  
    #     ipv4_address: 172.28.1.104  
    deploy:  
      resources:  
        limits:  
          cpus: '0.50'  
          memory: 1G  
        reservations:  
          cpus: '0.50'  
          memory: 1G  
  node5:  
    # build:  
    #   context: server  
    image: ***.azurecr.io/node-server  
    command: ./server -config config/cloud-neighbour-3.yaml -socket-address 0.0.0.0 -socket-port 8885  
    ports:  
      - 8885  
    # networks:  
    #   local:  
    #     ipv4_address: 172.28.1.105  
    deploy:  
      resources:  
        limits:  
          cpus: '0.50'  
          memory: 1G  
        reservations:  
          cpus: '0.50'  
          memory: 1G  
  node6:  
    # build:  
    #   context: server  
    image: ***.azurecr.io/node-server  
    command: ./server -config config/cloud-neighbour-3.yaml -socket-address 0.0.0.0 -socket-port 8886  
    ports:  
      - 8886  
    # networks:  
    #   local:  
    #     ipv4_address: 172.28.1.106  
    deploy:  
      resources:  
        limits:  
          cpus: '0.50'  
          memory: 1G  
        reservations:  
          cpus: '0.50'  
          memory: 1G  
  # client:  
  #   image: ***.azurecr.io/client  
  #   command: ./client  
  #   networks:  
  #     local:  
  #       ipv4_address: 172.28.1.200  
  #   deploy:  
  #     resources:  
  #       limits:  
  #         cpus: '0.50'  
  #         memory: 1G  
  
# networks:  
#   local:  
#     driver: bridge  
#     ipam:  
#      config:  
#        - subnet: 172.28.1.0/24  
#          gateway: 172.28.1.1  

I tried to remove the network on container to see if it is better, but not.
Please note, that I am using deploy.resources.limits on my docker-compose.yaml file to not exceed quota defined by Azure (4 CPU, 16G RAM).
Note alse, that I use the flag --verbose when using docker compose but it seems that I didn't have more logging..

Azure Container Registry
Azure Container Registry
An Azure service that provides a registry of Docker and Open Container Initiative images.
374 questions
Azure Container Instances
Azure Container Instances
An Azure service that provides customers with a serverless container experience.
627 questions
0 comments No comments
{count} votes

2 answers

Sort by: Most helpful
  1. deherman-MSFT 32,951 Reputation points Microsoft Employee
    2022-11-08T16:58:56.237+00:00

    @Bruno MC
    I understand you are trying to deploy 6 containers and seeing it fail with the above error. I believe this could be because you have the reservations equal to the limits. Please try to change this to see if it resolves your issue.

    From the documentation:
    "CPU and memory reservations and limits can be set in compose. Resource limits must be greater than reservation."

    See also this StackOverflow thread with a similar issue.

    Hope this helps! Let me know if you are still having issues and we can work with you to resolve them.

    -------------------------------

    Please don’t forget to "Accept the answer" and “up-vote” wherever the information provided helps you, this can be beneficial to other community members.

    0 comments No comments

  2. Bruno MC 1 Reputation point
    2022-11-15T15:30:42.433+00:00

    Hello, I have some news about this topic.

    Like @deherman-MSFT said, I was using the same limits and reservations. So I changed that to use only reservations but it changed anything.

    After that I tried to use another imag with the same configuration (reservations) and the deployment was working! So I was thinking that my problem is my container image that was too large (more than 1Go). So I reduced my golang image using practises explained in this blog : golang-small-docker-image

    Now my containers seems to be created, I don't have the timeout error on docker compose up. But unfortunatly the containers on azure still "Waiting", see image below.
    I don't have any logs. Do you have an idea of the reason or how to debug ?

    260577-aci-containers-waiting.png

    See my docker-compose.yaml below :

    version: "3"  
      
    services:  
      node1:  
        build:  
          context: server  
        image: ***.azurecr.io/node-server  
        command: ./server -config config/neighbour-1.yaml  
        networks:  
          local:  
            ipv4_address: 172.28.1.101  
        deploy:  
          resources:  
            reservations:  
              cpus: '0.25'  
              memory: 0.1gb  
      node2:  
        build:  
          context: server  
        image: ***.azurecr.io/node-server  
        command: ./server -config config/neighbour-2.yaml  
        networks:  
          local:  
            ipv4_address: 172.28.1.102  
        deploy:  
          resources:  
            reservations:  
              cpus: '0.25'  
              memory: 0.1gb  
      node3:  
        build:  
          context: server  
        image: ***.azurecr.io/node-server  
        command: ./server -config config/neighbour-3.yaml  
        networks:  
          local:  
            ipv4_address: 172.28.1.103  
        deploy:  
          resources:  
            reservations:  
              cpus: '0.25'  
              memory: 0.1gb  
      node4:  
        build:  
          context: server  
        image: ***.azurecr.io/node-server  
        command: ./server -config config/neighbour-4.yaml  
        networks:  
          local:  
            ipv4_address: 172.28.1.104  
        deploy:  
          resources:  
            reservations:  
              cpus: '0.25'  
              memory: 0.1gb  
      node5:  
        build:  
          context: server  
        image: ***.azurecr.io/node-server  
        command: ./server -config config/neighbour-5.yaml  
        networks:  
          local:  
            ipv4_address: 172.28.1.105  
        deploy:  
          resources:  
            reservations:  
              cpus: '0.25'  
              memory: 0.1gb  
      node6:  
        build:  
          context: server  
        image: ***.azurecr.io/node-server  
        command: ./server -config config/neighbour-6.yaml  
        networks:  
          local:  
            ipv4_address: 172.28.1.106  
        deploy:  
          resources:  
            reservations:  
              cpus: '0.25'  
              memory: 0.1gb  
      client:  
        build:  
          context: client  
        image: ***.azurecr.io/node-client  
        stdin_open: true # docker run -i  
        tty: true        # docker run -t  
        networks:  
          local:  
            ipv4_address: 172.28.1.50  
      # debug:  
      #   image: busybox  
      #   stdin_open: true # docker run -i  
      #   tty: true        # docker run -t  
      #   networks:  
      #     local:  
      #       ipv4_address: 172.28.1.51  
      
    networks:  
      local:  
        driver: bridge  
        ipam:  
         config:  
           - subnet: 172.28.1.0/24  
             gateway: 172.28.1.1