Note
Access to this page requires authorization. You can try signing in or changing directories.
Access to this page requires authorization. You can try changing directories.
The following is an example of a response body that is returned for the Get Big Data Cluster Information method, as defined in section 3.1.5.1.5.
The value of the following spec element is escaped by the server before it is sent to the client. The spec value is then unescaped by the client to create a valid JSON document.
-
{ "state": "Ready", "spec": "{ "apiVersion": "v1", "metadata": { "kind": "BigDataCluster", "name": "mssql-cluster" }, "spec": { "resources": { "gateway": { "clusterName": "mssql-cluster", "spec": { "replicas": 1, "docker": { "registry": "mcr.contoso.com", "repository": "mssql/bdc", "imageTag": "latest", "imagePullPolicy": "Always" }, "storage": { "data": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "15Gi" }, "logs": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "10Gi" } }, "endpoints": [ { "name": "Knox", "serviceType": "NodePort", "port": 30443, "dynamicDnsUpdate": true } ], "settings": { "gateway": { "gateway-site.gateway.httpclient.socketTimeout": "90s", "gateway-site.sun.security.krb5.debug": "true" } } } }, "appproxy": { "clusterName": "mssql-cluster", "spec": { "replicas": 1, "docker": { "registry": "mcr.contoso.com", "repository": "mssql/bdc", "imageTag": "latest", "imagePullPolicy": "Always" }, "storage": { "data": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "15Gi" }, "logs": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "10Gi" } }, "endpoints": [ { "name": "AppServiceProxy", "serviceType": "NodePort", "port": 30778, "dynamicDnsUpdate": true } ], "settings": {} } }, "storage-0": { "clusterName": "mssql-cluster", "metadata": { "kind": "Pool", "name": "default" }, "spec": { "type": "Storage", "replicas": 2, "docker": { "registry": "mcr.contoso.com", "repository": "mssql/bdc", "imageTag": "latest", "imagePullPolicy": "Always" }, "storage": { "data": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "15Gi" }, "logs": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "10Gi" } }, "settings": { "spark": { "includeSpark": "true", "yarn-site.yarn.nodemanager.resource.memory-mb": "18432", "yarn-site.yarn.scheduler.maximum-allocation-vcores": "6", "yarn-site.yarn.scheduler.maximum-allocation-mb": "18432", "spark-defaults-conf.spark.executor.instances": "3", "yarn-site.yarn.nodemanager.resource.cpu-vcores": "6", "spark-defaults-conf.spark.executor.cores": "1", "spark-defaults-conf.spark.driver.memory": "2g", "spark-defaults-conf.spark.driver.cores": "1", "yarn-site.yarn.scheduler.capacity.maximum-am-resource-percent": "0.3", "spark-defaults-conf.spark.executor.memory": "1536m", "capacity-scheduler.yarn.scheduler.capacity.maximum-applications": "10000", "capacity-scheduler.yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", "capacity-scheduler.yarn.scheduler.capacity.root.queues": "default", "capacity-scheduler.yarn.scheduler.capacity.root.default.capacity": "100", "capacity-scheduler.yarn.scheduler.capacity.root.default.user-limit-factor": "1", "capacity-scheduler.yarn.scheduler.capacity.root.default.maximum-capacity": "100", "capacity-scheduler.yarn.scheduler.capacity.root.default.state": "RUNNING", "capacity-scheduler.yarn.scheduler.capacity.root.default.maximum-application-lifetime": "-1", "capacity-scheduler.yarn.scheduler.capacity.root.default.default-application-lifetime": "-1", "capacity-scheduler.yarn.scheduler.capacity.node-locality-delay": "40", "capacity-scheduler.yarn.scheduler.capacity.rack-locality-additional-delay": "-1", "hadoop-env.HADOOP_HEAPSIZE_MAX": "2048", "yarn-env.YARN_RESOURCEMANAGER_HEAPSIZE": "2048", "yarn-env.YARN_NODEMANAGER_HEAPSIZE": "2048", "mapred-env.HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", "hive-env.HADOOP_HEAPSIZE": "2048", "livy-conf.livy.server.session.timeout-check": "true", "livy-conf.livy.server.session.timeout-check.skip-busy": "true", "livy-conf.livy.server.session.timeout": "2h", "livy-conf.livy.server.yarn.poll-interval": "500ms", "livy-env.LIVY_SERVER_JAVA_OPTS": "-Xmx2g", "spark-defaults-conf.spark.r.backendConnectionTimeout": "86400", "spark-history-server-conf.spark.history.fs.cleaner.maxAge": "7d", "spark-history-server-conf.spark.history.fs.cleaner.interval": "12h", "spark-env.SPARK_DAEMON_MEMORY": "2g", "yarn-site.yarn.log-aggregation.retain-seconds": "604800", "yarn-site.yarn.nodemanager.log-aggregation.compression-type": "gz", "yarn-site.yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds": "3600", "yarn-site.yarn.scheduler.minimum-allocation-mb": "512", "yarn-site.yarn.scheduler.minimum-allocation-vcores": "1", "yarn-site.yarn.nm.liveness-monitor.expiry-interval-ms": "180000" }, "sql": {}, "hdfs": { "hdfs-site.dfs.replication": "2", "hdfs-site.dfs.ls.limit": "500", "hdfs-env.HDFS_NAMENODE_OPTS": "-Dhadoop.security.logger=INFO,RFAS -Xmx2g", "hdfs-env.HDFS_DATANODE_OPTS": "-Dhadoop.security.logger=ERROR,RFAS -Xmx2g", "hdfs-env.HDFS_AUDIT_LOGGER": "INFO,RFAAUDIT", "core-site.hadoop.security.group.mapping.ldap.search.group.hierarchy.levels": "10", "core-site.fs.permissions.umask-mode": "077", "core-site.hadoop.security.kms.client.failover.max.retries": "20", "kms-site.hadoop.security.kms.encrypted.key.cache.size": "500", "zoo-cfg.tickTime": "2000", "zoo-cfg.initLimit": "10", "zoo-cfg.syncLimit": "5", "zoo-cfg.maxClientCnxns": "60", "zoo-cfg.minSessionTimeout": "4000", "zoo-cfg.maxSessionTimeout": "40000", "zoo-cfg.autopurge.snapRetainCount": "3", "zoo-cfg.autopurge.purgeInterval": "0", "zookeeper-java-env.JVMFLAGS": "-Xmx1G -Xms1G", "zookeeper-log4j-properties.zookeeper.console.threshold": "INFO" } } } }, "sparkhead": { "clusterName": "mssql-cluster", "spec": { "replicas": 2, "docker": { "registry": "mcr.contoso.com", "repository": "mssql/bdc", "imageTag": "latest", "imagePullPolicy": "Always" }, "storage": { "data": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "15Gi" }, "logs": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "10Gi" } }, "settings": { "hdfs": { "hdfs-site.dfs.replication": "2", "hdfs-site.dfs.ls.limit": "500", "hdfs-env.HDFS_NAMENODE_OPTS": "-Dhadoop.security.logger=INFO,RFAS -Xmx2g", "hdfs-env.HDFS_DATANODE_OPTS": "-Dhadoop.security.logger=ERROR,RFAS -Xmx2g", "hdfs-env.HDFS_AUDIT_LOGGER": "INFO,RFAAUDIT", "core-site.hadoop.security.group.mapping.ldap.search.group.hierarchy.levels": "10", "core-site.fs.permissions.umask-mode": "077", "core-site.hadoop.security.kms.client.failover.max.retries": "20", "kms-site.hadoop.security.kms.encrypted.key.cache.size": "500", "zoo-cfg.tickTime": "2000", "zoo-cfg.initLimit": "10", "zoo-cfg.syncLimit": "5", "zoo-cfg.maxClientCnxns": "60", "zoo-cfg.minSessionTimeout": "4000", "zoo-cfg.maxSessionTimeout": "40000", "zoo-cfg.autopurge.snapRetainCount": "3", "zoo-cfg.autopurge.purgeInterval": "0", "zookeeper-java-env.JVMFLAGS": "-Xmx1G -Xms1G", "zookeeper-log4j-properties.zookeeper.console.threshold": "INFO" }, "spark": { "yarn-site.yarn.nodemanager.resource.memory-mb": "18432", "yarn-site.yarn.scheduler.maximum-allocation-vcores": "6", "yarn-site.yarn.scheduler.maximum-allocation-mb": "18432", "spark-defaults-conf.spark.executor.instances": "3", "yarn-site.yarn.nodemanager.resource.cpu-vcores": "6", "spark-defaults-conf.spark.executor.cores": "1", "spark-defaults-conf.spark.driver.memory": "2g", "spark-defaults-conf.spark.driver.cores": "1", "yarn-site.yarn.scheduler.capacity.maximum-am-resource-percent": "0.3", "spark-defaults-conf.spark.executor.memory": "1536m", "capacity-scheduler.yarn.scheduler.capacity.maximum-applications": "10000", "capacity-scheduler.yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", "capacity-scheduler.yarn.scheduler.capacity.root.queues": "default", "capacity-scheduler.yarn.scheduler.capacity.root.default.capacity": "100", "capacity-scheduler.yarn.scheduler.capacity.root.default.user-limit-factor": "1", "capacity-scheduler.yarn.scheduler.capacity.root.default.maximum-capacity": "100", "capacity-scheduler.yarn.scheduler.capacity.root.default.state": "RUNNING", "capacity-scheduler.yarn.scheduler.capacity.root.default.maximum-application-lifetime": "-1", "capacity-scheduler.yarn.scheduler.capacity.root.default.default-application-lifetime": "-1", "capacity-scheduler.yarn.scheduler.capacity.node-locality-delay": "40", "capacity-scheduler.yarn.scheduler.capacity.rack-locality-additional-delay": "-1", "hadoop-env.HADOOP_HEAPSIZE_MAX": "2048", "yarn-env.YARN_RESOURCEMANAGER_HEAPSIZE": "2048", "yarn-env.YARN_NODEMANAGER_HEAPSIZE": "2048", "mapred-env.HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", "hive-env.HADOOP_HEAPSIZE": "2048", "livy-conf.livy.server.session.timeout-check": "true", "livy-conf.livy.server.session.timeout-check.skip-busy": "true", "livy-conf.livy.server.session.timeout": "2h", "livy-conf.livy.server.yarn.poll-interval": "500ms", "livy-env.LIVY_SERVER_JAVA_OPTS": "-Xmx2g", "spark-defaults-conf.spark.r.backendConnectionTimeout": "86400", "spark-history-server-conf.spark.history.fs.cleaner.maxAge": "7d", "spark-history-server-conf.spark.history.fs.cleaner.interval": "12h", "spark-env.SPARK_DAEMON_MEMORY": "2g", "yarn-site.yarn.log-aggregation.retain-seconds": "604800", "yarn-site.yarn.nodemanager.log-aggregation.compression-type": "gz", "yarn-site.yarn.nodemanager.log-aggregation.roll-monitoring-interval-seconds": "3600", "yarn-site.yarn.scheduler.minimum-allocation-mb": "512", "yarn-site.yarn.scheduler.minimum-allocation-vcores": "1", "yarn-site.yarn.nm.liveness-monitor.expiry-interval-ms": "180000" } } } }, "data-0": { "clusterName": "mssql-cluster", "metadata": { "kind": "Pool", "name": "default" }, "spec": { "type": "Data", "replicas": 2, "docker": { "registry": "mcr.contoso.com", "repository": "mssql/bdc", "imageTag": "latest", "imagePullPolicy": "Always" }, "storage": { "data": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "15Gi" }, "logs": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "10Gi" } }, "settings": { "sql": {} } } }, "compute-0": { "clusterName": "mssql-cluster", "metadata": { "kind": "Pool", "name": "default" }, "spec": { "type": "Compute", "replicas": 1, "docker": { "registry": "mcr.contoso.com", "repository": "mssql/bdc", "imageTag": "latest", "imagePullPolicy": "Always" }, "storage": { "data": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "15Gi" }, "logs": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "10Gi" } }, "settings": { "sql": {} } } }, "master": { "clusterName": "mssql-cluster", "metadata": { "kind": "Pool", "name": "default" }, "spec": { "type": "Master", "replicas": 1, "docker": { "registry": "mcr.contoso.com", "repository": "mssql/bdc", "imageTag": "latest", "imagePullPolicy": "Always" }, "storage": { "data": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "15Gi" }, "logs": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "10Gi" } }, "endpoints": [ { "name": "Master", "serviceType": "NodePort", "port": 31433, "dynamicDnsUpdate": true } ], "settings": { "sql": { "hadr.enabled": "false" } } } }, "nmnode-0": { "clusterName": "mssql-cluster", "spec": { "replicas": 2, "docker": { "registry": "mcr.contoso.com", "repository": "mssql/bdc", "imageTag": "latest", "imagePullPolicy": "Always" }, "storage": { "data": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "15Gi" }, "logs": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "10Gi" } }, "settings": { "hdfs": { "hdfs-site.dfs.replication": "2", "hdfs-site.dfs.ls.limit": "500", "hdfs-env.HDFS_NAMENODE_OPTS": "-Dhadoop.security.logger=INFO,RFAS -Xmx2g", "hdfs-env.HDFS_DATANODE_OPTS": "-Dhadoop.security.logger=ERROR,RFAS -Xmx2g", "hdfs-env.HDFS_AUDIT_LOGGER": "INFO,RFAAUDIT", "core-site.hadoop.security.group.mapping.ldap.search.group.hierarchy.levels": "10", "core-site.fs.permissions.umask-mode": "077", "core-site.hadoop.security.kms.client.failover.max.retries": "20", "kms-site.hadoop.security.kms.encrypted.key.cache.size": "500", "zoo-cfg.tickTime": "2000", "zoo-cfg.initLimit": "10", "zoo-cfg.syncLimit": "5", "zoo-cfg.maxClientCnxns": "60", "zoo-cfg.minSessionTimeout": "4000", "zoo-cfg.maxSessionTimeout": "40000", "zoo-cfg.autopurge.snapRetainCount": "3", "zoo-cfg.autopurge.purgeInterval": "0", "zookeeper-java-env.JVMFLAGS": "-Xmx1G -Xms1G", "zookeeper-log4j-properties.zookeeper.console.threshold": "INFO" } } } }, "zookeeper": { "clusterName": "mssql-cluster", "spec": { "replicas": 3, "docker": { "registry": "mcr.contoso.com", "repository": "mssql/bdc", "imageTag": "latest", "imagePullPolicy": "Always" }, "storage": { "data": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "15Gi" }, "logs": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "10Gi" } }, "settings": { "hdfs": { "hdfs-site.dfs.replication": "2", "hdfs-site.dfs.ls.limit": "500", "hdfs-env.HDFS_NAMENODE_OPTS": "-Dhadoop.security.logger=INFO,RFAS -Xmx2g", "hdfs-env.HDFS_DATANODE_OPTS": "-Dhadoop.security.logger=ERROR,RFAS -Xmx2g", "hdfs-env.HDFS_AUDIT_LOGGER": "INFO,RFAAUDIT", "core-site.hadoop.security.group.mapping.ldap.search.group.hierarchy.levels": "10", "core-site.fs.permissions.umask-mode": "077", "core-site.hadoop.security.kms.client.failover.max.retries": "20", "kms-site.hadoop.security.kms.encrypted.key.cache.size": "500", "zoo-cfg.tickTime": "2000", "zoo-cfg.initLimit": "10", "zoo-cfg.syncLimit": "5", "zoo-cfg.maxClientCnxns": "60", "zoo-cfg.minSessionTimeout": "4000", "zoo-cfg.maxSessionTimeout": "40000", "zoo-cfg.autopurge.snapRetainCount": "3", "zoo-cfg.autopurge.purgeInterval": "0", "zookeeper-java-env.JVMFLAGS": "-Xmx1G -Xms1G", "zookeeper-log4j-properties.zookeeper.console.threshold": "INFO" } } } } }, "services": { "sql": { "resources": [ "master", "compute-0", "data-0", "storage-0" ], "settings": {} }, "hdfs": { "resources": [ "nmnode-0", "zookeeper", "storage-0", "sparkhead" ], "settings": {} }, "spark": { "resources": [ "sparkhead", "storage-0" ], "settings": { "yarn-site.yarn.nodemanager.resource.memory-mb": "18432", "yarn-site.yarn.scheduler.maximum-allocation-vcores": "6", "yarn-site.yarn.scheduler.maximum-allocation-mb": "18432", "spark-defaults-conf.spark.executor.instances": "3", "yarn-site.yarn.nodemanager.resource.cpu-vcores": "6", "spark-defaults-conf.spark.executor.cores": "1", "spark-defaults-conf.spark.driver.memory": "2g", "spark-defaults-conf.spark.driver.cores": "1", "yarn-site.yarn.scheduler.capacity.maximum-am-resource-percent": "0.3", "spark-defaults-conf.spark.executor.memory": "1536m" } } }, "docker": { "registry": "mcr.contoso.com", "repository": "mssql/bdc", "imageTag": "latest", "imagePullPolicy": "Always" }, "storage": { "data": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "15Gi" }, "logs": { "className": "local-storage", "accessMode": "ReadWriteOnce", "size": "10Gi" } } } }" }