CREATE TABLE ManagedUnpartitioned ( column1 STRING ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE LOCATION '/demo/managedunpartitioned/'; CREATE TABLE ManagedPartitioned ( column1 STRING ) PARTITIONED BY (partition1 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE LOCATION '/demo/managedpartitioned/'; CREATE EXTERNAL TABLE ExternalUnpartitioned ( column1 STRING ) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE LOCATION '/demo/externalunpartitioned/'; CREATE EXTERNAL TABLE ExternalPartitioned ( column1 STRING ) PARTITIONED BY (partition1 STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' STORED AS TEXTFILE LOCATION '/demo/externalpartitioned/'; INSERT INTO TABLE ManagedUnpartitioned VALUES ('this is a test'), ('this is also a test'); INSERT INTO TABLE ManagedPartitioned PARTITION(partition1='A') VALUES ('this is a test'); INSERT INTO table ManagedPartitioned PARTITION(partition1='B') VALUES ('this is also a test'); INSERT INTO TABLE ExternalUnpartitioned VALUES ('this is a test'), ('this is also a test'); INSERT INTO TABLE ExternalPartitioned PARTITION(partition1='A') VALUES ('this is a test'); INSERT INTO TABLE ExternalPartitioned PARTITION(partition1='B') VALUES ('this is also a test');

EXPORT TABLE tablename TO 'targeted export location';

EXPORT TABLE ManagedUnpartitioned TO '/export/managedunpartitioned'; EXPORT TABLE ManagedUnpartitioned TO 'hdfs://master3:8020/export/managedunpartitioned'; EXPORT TABLE ManagedPartitioned TO '/export/managedpartitioned'; EXPORT TABLE ManagedPartitioned TO 'hdfs://master3:8020/export/managedpartitioned'; EXPORT TABLE ExternalUnpartitioned TO '/export/externalunpartitioned'; EXPORT TABLE ExternalUnpartitioned TO 'hdfs://master3:8020/export/externalunpartitioned'; EXPORT TABLE ExternalPartitioned TO '/export/externalpartitioned'; EXPORT TABLE ExternalPartitioned TO 'hdfs://master3:8020/export/externalpartitioned';

EXPORT TABLE ManagedPartitioned PARTITION (partition1='A') TO '/export/managedpartitioned/a'; EXPORT TABLE ManagedPartitioned PARTITION (partition1='B') TO '/export/managedpartitioned/b'; EXPORT TABLE ExternalPartitioned PARTITION (partition1='A') TO '/export/externalpartitioned/a'; EXPORT TABLE ExternalPartitioned PARTITION (partition1='B') TO '/export/externalpartitioned/b';

IMPORT TABLE tablename FROM 'targeted import location'; IMPORT EXTERNAL TABLE tablename FROM 'targeted import location';

• The table being created should not already exist as this command will attempt to create a new table using information in the _metadata file.
• The name of the table being created does not need to match the name of the original (exported) table.
• The table being created can be either managed or external, regardless of the status of the original table.

-- original export command -- EXPORT TABLE ManagedPartitioned TO '/export/managedpartitioned'; IMPORT TABLE managedpartitioned_imported FROM '/export/managedpartitioned'; IMPORT EXTERNAL TABLE managedpartitioned_imported FROM '/export/managedpartitioned'; IMPORT TABLE managedpartitioned_imported FROM 'hdfs://master3:8020/export/managedpartitioned'; IMPORT EXTERNAL TABLE managedpartitioned_imported FROM 'hdfs://master3:8020/export/managedpartitioned';

-- original export commands -- EXPORT TABLE ManagedPartitioned PARTITION (partition1='A') TO '/export/managedpartitioned/a'; -- EXPORT TABLE ManagedPartitioned PARTITION (partition1='B') TO '/export/managedpartitioned/b'; IMPORT TABLE managedpartitioned_imported PARTITION (partition1='A') FROM '/export/managedpartitioned/a'; IMPORT TABLE managedpartitioned_imported PARTITION (partition1='B') FROM '/export/managedpartitioned/b'; IMPORT EXTERNAL TABLE managedpartitioned_imported PARTITION (partition1='A') FROM '/export/managedpartitioned/a'; IMPORT EXTERNAL TABLE managedpartitioned_imported PARTITION (partition1='B') FROM '/export/managedpartitioned/b';

IMPORT TABLE managedpartitioned_importedrelo FROM '/export/managedpartitioned' LOCATION '/relo/managedpartitioned'; IMPORT EXTERNAL TABLE managedpartitioned_importedrelo FROM '/export/managedpartitioned' LOCATION '/relo/managedpartitioned';

IMPORT TABLE managedpartitioned_importedrelo PARTITION (partition1='A') FROM '/export/managedpartitioned/a' LOCATION '/relo/managedpartitioned_importedrelo'; IMPORT TABLE managedpartitioned_importedrelo PARTITION (partition1='B') FROM '/export/managedpartitioned/b' LOCATION '/relo/managedpartitioned_importedrelo';

EXPORT TABLE ManagedUnpartitioned TO 'wasb://mycontainer@mystorageaccount.blob.core.windows.net/export/managedunpartitioned'; IMPORT TABLE ManagedUnpartitioned_imported FROM 'wasb://mycontainer@mystorageaccount.blob.core.windows.net/export/managedunpartitioned';

IMPORT TABLE ManagedUnpartitioned_imported FROM 'wasb://mycontainer@mystorageaccount.blob.core.windows.net/export/managedunpartitioned' LOCATION '/relo/managedunpartitioned_importedrelo';

1. Login to the Ambari portal and navigate to the (default) dashboard
2. Select HDFS from the left-hand navigation
3. On the resulting HDFS page, click on the Configs tab
4. On the resulting page, select the Advanced option
5. Scroll down and expand the Custom core-site node
6. Select Add Property… from the bottom of the expanded node
7. Enter fs.azure.account.key.<account name>.blob.core.windows.net, substituting the name of the Storage account for <account name>, as the Name of the property
8. Enter the Account key as the Value of the property
9. Click the Add button and verify the new property appears under the Custom core-site node
10. Locate the notification bar at the top of the current page
11. Click the Save button on the notification bar to push changes to the cluster
12. Follow any remaining prompts to complete the Save process

• wasb://<container name>@<account name>.blob.core.windows.net/
• wasbs://<container name>@<account name>.blob.core.windows.net/

   12345678
   12345678
   12345678

   USE DATABASE [master];
   REFERENCE ASSEMBLY [FixedWidthExtractor];
   @tbl =
      EXTRACT
         col1 int,
         col2 float,
         col3 decimal,
         col4 byte[]
      FROM "/fixedwidth/input.txt"
      USING new MyCustomExtractors.FixedWidthExtractor (
         new SQL.MAP<string, string> {
            {"col1","3"},
            {"col2","2"},
            {"col3","1"},
            {"col4","2"}
         }
      );
   OUTPUT @tbl
   TO "/fixedwidth/output.txt"
   USING Outputters.Csv();

   namespace MyCustomExtractors
   {
      public class FixedWidthExtractor : IExtractor
      {
         … //coding magic happens here
      }
   }

   public FixedWidthExtractor(SqlMap<string, string> col_widths, Encoding encoding = null, string row_delim = "\r\n")
   {
      this._encoding = ((encoding == null) ? Encoding.UTF8 : encoding);
      this._row_delim = this._encoding.GetBytes(row_delim);
      this._col_widths = col_widths;
   }

   public override IEnumerable<IRow> Extract(IUnstructuredReader input, IUpdatableRow output)
   {
      ... //more code magic here
   }

   public override IEnumerable<IRow> Extract(IUnstructuredReader input, IUpdatableRow output)
   {
      foreach (Stream currentline in input.Split(this._row_delim))
      {
         using (StreamReader lineReader = new StreamReader(currentline, this._encoding))
         {
            string line = lineReader.ReadToEnd();
            … //the good stuff is coming here
         }
      }
   }

   //read new line of input
   int start_parse = 0;

   //for each column
   int i = 0;
   foreach (var col_width in this._col_widths)
   {
      //read chars associated with fixed-width column
      int chars_to_read = int.Parse(col_width.Value);
      string value = line.Substring(start_parse, chars_to_read);


      //assign value to output (w/ appropriate type)
      switch (output.Schema[i].Type.Name)
      {
         case ("String"):
            output.Set(i, value);
            break;
         case ("Int32"):
            output.Set(i, Int32.Parse(value));
            break;
         ... //shortened for readability
         default:
            throw (new Exception("Unknown data type specified: "+output.Schema[i].Type.Name));
      }

      //move to start of next column
      start_parse += chars_to_read;
      i++;
   }


   //send output
   yield return output.AsReadOnly();

• Nielsen (Consumer Data)
• IRI (Consumer Data)
• Acxiom (Consumer & Online Data)
• BlueKai (Online Data)
• Melissa Data (Consumer Data)
• Dun & Bradstreet (Business Data)
• OnTerra Systems (Geographical Data)
• Trimble (Geographical Data)

• US Census Data
• US Internal Revenue Service
• US Dept of Commerce, International Trade Administration

• Azure Data Marketplace
• AWS Public Data Sets
• Google Public Data
• Qlik Data Market

• Reddit, Data is Beautiful
• KDD Cup Datasets
• Kaggle
• Open Data Sets on GitHub

1. Deploy Azure HDInsight cluster to a Virtual Network accessible by Drill
2. Configure HBase access within Drill
3. Confirm HBase access within Drill

1. Open a browser to https://myclustername.azurehdinsight.net with appropriate substitution for myclustername
2. At the prompt, login using the HTTP user name and password established when you configured the HDInsight cluster
3. Within the Ambari interface, go to Services | HBase | Configs | Advanced
4. Under the Advanced hbase-site subheading, copy the value for hbase.zookeeper.quorum and note the value for zookeeper.znode.parent, which should be /hbase-unsecure.

{ "type": "hbase", "config": { "hbase.zookeeper.quorum": "zk1-drillh.h3yoplooa.dx.internal.cloudapp.net,zk3-drillh.h3yoplooa.dx.internal.cloudapp.net,zk2-drillh.h3yoplooa.dx.internal.cloudapp.net", "hbase.zookeeper.property.clientPort": "2181", "zookeeper.znode.parent": "/hbase-unsecure" }, "size.calculator.enabled": false, "enabled": true }

SELECT * FROM hbase.`ambarismoketest`;

1. Deploy Azure HDInsight cluster to a Virtual Network accessible by Drill
2. Configure WASB access within Drill
3. Deploy required SerDes to Drill servers
4. Configure Hive access within Drill
5. Confirm Hive access within Drill

cd /drill/current/jars/3rdparty/ sudo scp sshusername@myclustername-ssh.azurehdinsight.net:/usr/hdp/current/hive-client/regepx-1.3.jar .

1. Open a browser to https://myclustername.azurehdinsight.net with appropriate substitution for myclustername
2. At the prompt, login using the HTTP user name and password established when you configured the cluster
3. Within the Ambari interface, go to Services | Hive  | Configs | Advanced
4. Under the General subheading, locate and record the values for hive.metastore.uris and hive.metastore.warehouse.dir
5. Navigate to Services | HDFS | Configs | Advanced
6. Under the Advanced core-site subheading, locate and record the value for fs.defaultFS

{ "type": "hive", "enabled": true, "configProps": { "hive.metastore.uris": "thrift://hn0-myclust.h3yob50453plbaooa.dx.internal.cloudapp.net:9083,thrift://hn1-myclust.h3yob50453plbaooa.dx.internal.cloudapp.net:9083", "hive.metastore.warehouse.dir": "/hive/warehouse", "fs.default.name": "wasb://mycontainer@mydatafiles.blob.core.windows.net/", "hive.metastore.sasl.enabled": "false" } }

SELECT * FROM hive.`hivesampletable` LIMIT 10;