1,Shoes,2,99.99,10
1,Pants,3,59,34,32
2,Camera,1,999.00,Canon,70D
2,Lens,1,999.99,Canon,100mm.Macro,1.4
2,Lens,1,459.99,Sigma,28-85mm,Macro/Zoom,2.8
3,Camera,1,745,Sony,RX-100-II
3,Shoes,1,69.99,13

Writing scripts to extract data

DECLARE @input string = "/Samples/Blogs/MRys/FlexExtractor/OrderData.csv";

// Usage with all columns known
@data = EXTRACT orderid int, producttype string, orderamount int, itemprice decimal,
                c1 string, c2 string, c3 string, c4 string
        FROM @input
        USING new FlexibleSchemaExtractor.FlexExtractor();

// product type Camera
@cameras =
    SELECT orderid,
           orderamount,
           itemprice,
           c1 AS make,
           c2 AS model
    FROM @data
    WHERE producttype == "Camera";

OUTPUT @cameras
TO "/output/cameras.csv"
USING Outputters.Csv();

// product type Lens
@lenses =
    SELECT orderid,
           orderamount,
           itemprice,
           c1 AS make,
           c2 AS focallength,
           c3 AS lenstype,
           c4 == null ? (decimal?) null : Decimal.Parse(c4) AS aperture
    FROM @data
    WHERE producttype == "Lens";

OUTPUT @lenses
TO "/output/lenses.csv"
USING Outputters.Csv();

Writing the Flexible Schema Extractor UDO

using Microsoft.Analytics.Interfaces;
using Microsoft.Analytics.Types.Sql;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;

namespace FlexibleSchemaExtractor
{
    public class FlexExtractor : IExtractor
    {
        private Encoding _encoding;
        private byte[] _row_delim;
        private string _col_delim;

        public FlexExtractor(Encoding encoding = null, string row_delim = "\r\n", string col_delim = ",")
        {
            this._encoding = ((encoding == null) ? Encoding.UTF8 : encoding);
            this._row_delim = this._encoding.GetBytes(row_delim);
            this._col_delim = col_delim;
        }

        public override IEnumerable<IRow> Extract(IUnstructuredReader input, IUpdatableRow output)
        {
            var colsInSchema = output.Schema.Count;

            // let's check global assumptions
            // - first 4 provided columns are int, string, int, decimal.

            if (   output.Schema[0].Type != typeof(System.Int32) 
                || output.Schema[1].Type != typeof(System.String)
                || output.Schema[2].Type != typeof(System.Int32)
                || output.Schema[3].Type != typeof(System.Decimal)
               )
            {
                throw new Exception("First 4 columns are not of expected types int32, string, int32, decimal.");
            }

            foreach (Stream currentline in input.Split(this._row_delim))
            {
                using (StreamReader lineReader = new StreamReader(currentline, this._encoding))
                {
                    string[] columns = lineReader.ReadToEnd().Split( new string[] { this._col_delim }
                                                                   , StringSplitOptions.None);
                    var colsInData = columns.Length;

                    // let's check row level assumptions
                    // - if less columns are specified, then last column needs to be of type SqlMap<Int32, string>

                    if (   colsInData > colsInSchema 
                        && output.Schema[colsInSchema - 1].Type != typeof(SqlMap<Int32, string>))
                    {
                        throw new Exception(
                              "Too many columns detected and last column is not of type SqlMap<Int32,string>. " 
                            + "Add a final column of type SqlMap<Int32,string&ht; into your extract schema.");
                    }
                    // Set first 4 fixed columns
                    output.Set<Int32>(0, Int32.Parse(columns[0]));
                    output.Set<String>(1, columns[1]);
                    output.Set<Int32>(2, Int32.Parse(columns[2]));
                    output.Set<Decimal>(3, Decimal.Parse(columns[3]));

                    // Fill all remaining columns except the last which may be a map
                    for (int i = 4; i < Math.Min(colsInData, colsInSchema) - 1; i++)
                    {
                        output.Set<String>(i, columns[i]);
                    }

                    // Now handle last column: if it is a map
                    if (   colsInData >= colsInSchema 
                        && output.Schema[colsInSchema - 1].Type == typeof(SqlMap<Int32,string>))
                    {
                        var sqlmap = new Dictionary<Int32,string>();
                        for (int j = colsInSchema - 1; j < colsInData; j++)
                        {
                            sqlmap.Add(j - colsInSchema + 1, columns[j]);
                        }
                        output.Set<SqlMap<Int32,string>>(colsInSchema - 1, new SqlMap<Int32,string>(sqlmap));
                    }
                    // Now handle last column: if it is not a map
                    else if (colsInData == Math.Min(colsInData, colsInSchema))
                    {
                        output.Set<string>(colsInData - 1, columns[colsInData - 1]);
                    }

                    yield return output.AsReadOnly();
                }
            }
        }
    }
}

Extracting Flexible columns with a SqlMap

@data = EXTRACT orderid int, producttype string, orderamount int, itemprice decimal,
                c1 string, map SqlMap<Int32,string>
        FROM @input
        USING new FlexibleSchemaExtractor.FlexExtractor();

// product type Camera
@cameras =
    SELECT orderid,
           orderamount,
           itemprice,
           c1 AS make,
           map[0] AS model
    FROM @data
    WHERE producttype == "Camera";

OUTPUT @cameras
TO "/output/cameras2.csv"
USING Outputters.Csv();

// product type Lens
@lenses =
    SELECT orderid,
           orderamount,
           itemprice,
           c1 AS make,
           map[0] AS focallength,
           map[1] AS lenstype,
           map[2] == null ? (decimal?) null : Decimal.Parse(map[2]) AS aperture
    FROM @data
    WHERE producttype == "Lens";

OUTPUT @lenses
TO "/output/lenses2.csv"
USING Outputters.Csv();

// remaining product types (serialize map generically)
@others =
    SELECT orderid,
           producttype,
           orderamount,
           itemprice,
           c1,
           map == null ? (string) null 
                       : string.Join(" ; ", 
                                     from p in map 
                                     select string.Format("{0}{1}{2}", p.Key, " : ", p.Value)) AS map 
    FROM @data
    WHERE producttype NOT IN ("Camera", "Lens");

OUTPUT @others
TO "/users/herring/others.csv"
USING Outputters.Csv();

Conclusion

Given a log file that contains time ranges in the form (begin, end) for a user-name, I would like to merge overlapping ranges for each user.

Start Time - End Time - User Name
5:00 AM - 6:00 AM - ABC
5:00 AM - 6:00 AM - XYZ
8:00 AM - 9:00 AM - ABC
8:00 AM - 10:00 AM - ABC
10:00 AM - 2:00 PM - ABC
7:00 AM - 11:00 AM - ABC
9:00 AM - 11:00 AM - ABC
11:00 AM - 11:30 AM - ABC
11:40 PM - 11:59 PM - FOO
11:50 PM - 0:40 AM - FOO

Start Time - End Time - User Name
5:00 AM - 6:00 AM - ABC
5:00 AM - 6:00 AM - XYZ
7:00 AM - 2:00 PM - ABC
11:40 PM - 0:40 AM - FOO

Analysis of the problem and possible solution

@in = EXTRACT begin DateTime, end DateTime, user string
FROM "/Samples/Blogs/MRys/Ranges/ranges.txt"
USING Extractors.Text(delimiter:'-');

@r = REDUCE @in ON user
PRODUCE begin DateTime, end DateTime, user string
USING new ReduceSample.RangeReducer();

OUTPUT @r
TO "/temp/result.csv"
USING Outputters.Csv();

First attempt at writing the range reducer

using Microsoft.Analytics.Interfaces;
using Microsoft.Analytics.Types.Sql;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace ReduceSample
{
    public class RangeReducer : IReducer
    {
        public override IEnumerable<IRow> Reduce(IRowset input, IUpdatableRow output)
        {
         // Insert your code to reduce
        } // Reduce
    } // RangeReducer
} // ReduceSample

public override IEnumerable<IRow> Reduce(IRowset input, IUpdatableRow output)
{
  foreach (var row in input.Rows.OrderBy(x => x.Get<DateTime>("begin")))
  {
   // perform the range merging
  } // foreach
} // Reduce

E_RUNTIME_USER_UNHANDLED_EXCEPTION_FROM_USER_CODE
An unhandled exception from user code has been reported.
Unhandled exception from user code: "Access an expired row using Get method, column: begin, index: 0. Please clone row when iterating row set"

The final version of the range reducer

public override IEnumerable<IRow> Reduce(IRowset input, IUpdatableRow output)
{
  // Init aggregation values
  int i = 0;
  var begin = DateTime.MaxValue; // Dummy value to make compiler happy
  var end = DateTime.MinValue; // Dummy value to make compiler happy

  // requires that the reducer is PRESORTED on begin and READONLY on the reduce key.
  foreach (var row in input.Rows)
  {
    // Initialize the first interval with the first row if i is 0
    if (i == 0)
    {
      i++; // mark that we handled the first row
      begin = row.Get<DateTime>("begin");
      end = row.Get<DateTime>("end");
      // If the end is just a time and not a date, it can be earlier than the begin, indicating it is on the next day.
      // This let's fix up the end to the next day in that case
      if (end < begin) { end = end.AddDays(1); }
    }
    else // handle the remaining rows
    {
      var b = row.Get<DateTime>("begin");
      var e = row.Get<DateTime>("end");
      // fix up the date if end is earlier than begin
      if (e < b) { e = e.AddDays(1); }

      // if the begin is still inside the interval, increase the interval if it is longer
      if (b <= end)
      {
        // if the new end time is later than the current, extend the interval
        if (e > end) { end = e; }
      }
      else // output the previous interval and start a new one
      {
        output.Set<DateTime>("begin", begin);
        output.Set<DateTime>("end", end);
        yield return output.AsReadOnly();
        begin = b; end = e;
      } // if
    } // if
  } // foreach

  // now output the last interval
  output.Set<DateTime>("begin", begin);
  output.Set<DateTime>("end", end);
  yield return output.AsReadOnly();
} // Reduce

@in = EXTRACT begin DateTime, end DateTime, user string
FROM "/Samples/Blogs/MRys/Ranges/ranges.txt"
USING Extractors.Text(delimiter:'-');

@r = REDUCE @in PRESORT begin ON user
PRODUCE begin DateTime, end DateTime, user string
READONLY user
USING new ReduceSample.RangeReducer();

OUTPUT @r
TO "/temp/result.csv"
USING Outputters.Csv();

Final note: Recursive vs non-recursive reducers

namespace ReduceSample
{
    [SqlUserDefinedReducer(IsRecursive = true)]
    public class RangeReducer : IReducer
    {
        public override IEnumerable<IRow> Reduce(IRowset input, IUpdatableRow output)
        {
         // Insert your code to reduce
        } // Reduce
    } // RangeReducer
} // ReduceSample

• Introducing U-SQL – A Language that makes Big Data Processing Easy
• Writing and Using Custom Code in U-SQL – User-Defined Functions
• Big Data - Making Big Data Batch Analytics Easier Using U-SQL

• ADL and U-SQL feedback and feature wishes
• ADLA and U-SQL MSDN Forum
• U-SQL related questions on stackoverflow
• The U-SQL Github site that contains many of the samples
• U-SQL Reference documentation
• My tweet stream (use #USQL to tweet about U-SQL since Twitter doesn't handle '-' in tags well)
• And last but not least the Big Data team blog (where I will move some of the blog posts that I develop here on U-SQL and ADLA).

Wednesday, September 19th, 9:45am to 11am, room 709:
DBA-317-M The Next Release of Microsoft SQL Server: Beyond Relational
(co-presenting with Kevin Farlee)
Main focus will be the new features in the areas of unstructured and semistructured data management in SQL Server 2008.

Wednesday, September 19th, 1:30pm to 2:45pm, room 709:
AD-407-M Deep Dive into XQuery and XML in Microsoft SQL Server: Common Problems & Best Practice Solutions
This is the same presentation I gave at TechEd 2007, a bit shortened to fit into the timeslot. Focus is on SQL Server 2005 XQuery and XML optimizations, however they also will apply to SQL Server 2008.

Thursday, September 20th, 1:30pm to 2:45pm, room 709:
AD-312-M Building Spatial Application with Microsoft SQL Server
(co-presenting with Isaac Kunen)
This will be an updated presentation of the chalktalk that I gave at TechEd 2007. Main focus is that the demo runs now on the bits that will be shipping :).

We merged the Friday 4pm presentation on managing unstructured and semistructured data (AD-313-M) with the presentation on "Beyond Relational" that Kevin and I will give on Wednesday. Primarily because the two presentations had a large overlap and neither Shan nor Srini were able to attend.

What does this bring for XML Support and other interesting programmability support, you may ask. Well, we are doing a bit of XML (not quiet as much as you may would have liked, but we need to also invest in other areas ;-) ), added a new MERGE statement, finally added some more usable date and time datatypes, work on adding support for spatial datatypes and indexing and a slew of other interesting, cool features.

I personally have been working on XML (obviously) and spatial indexing, besides being involved in one way or another with many of the programmability features, and - yes, that's the most important change for the product ;-) - the change of @@version from 9.0 to 10.0 :-).

I am at TechEd as well and will give the following breakout and chalk-talk presentations on XML and Spatial:

Wednesday, June 6 10:15 AM - 11:30 AM, Blue Theater 12:
DAT11-TLC - XML Q&A
Speaker(s): Peter DeBetta, Michael Rys
Two experts, one place to get your questions on XML in SQL Server answered. Interested in what's new in SQL Server 2008 ("Katmai"), or the new FOR XML PATH? Want to know when to use XML? How to index it? Need to know how to write an XQuery? Come to this session, and Michael and Peter will see that you get an answer.

Thursday, June 7 9:45 AM - 11:00 AM, Blue Theater 12:
DAT17-TLC - Spatial Support in Microsoft SQL Server
Speaker(s): Michael Rys, Michael Wang
The next version of SQL Server will include provide support for spatial data. This chalk talk presents the new spatial datatypes and indexing mechanisms so you can fully take advantage of the new capabilities that are being offered.

Friday, June 8 1:00 PM - 2:15 PM, S210 E:
DAT422 - Deep Dive into XQuery and XML in Microsoft SQL Server: Common Problems and Best Practice Solutions
Speaker(s): Michael Rys
This presentation provides a deep dive into the XML, XQuery, and XML indexing support in SQL Server 2005. It uses example scenarios that customers have come across when using XML and XQuery to show how to use optimize XQuery expressions, and provides the best practices on how to address the encountered issues.

Of additional interest should be the following presentations:

Today, Monday, June 4 3:00 PM - 4:15 PM, S320 A:
DAT201 - The Next Release of Microsoft SQL Server: Overview
Speaker(s): David Campbell
This session provides an overview of the next release of SQL Server that is currently under development. We cover the core value-proposition, major themes and scenarios, and some specific improvements. We also cover the new development processes being used to build this release, the release timeline and the disclosure calendar.

Tuesday, June 5 2:45 PM - 4:00 PM, N320 E:
DAT202 - The Next Release of Microsoft SQL Server: Beyond Relational  
Speaker(s): Shishir Mehrotra
As the data your applications work with changes from “words and numbers” to “sights and sounds”, we are evolving our data platform to go beyond relational data, beyond OLAP, to truly support all of the digital data types of the future. We will strive to deliver the best platform for integrated storage, and advanced applications such as spatial data, while also making it dramatically easier for you to build data-driven applications, without needing to invest significant resources in bridging the gap between data and programming language data structures. Come learn about the new enhancements in the next release of SQL Server that will help you manage all types of data.

Tuesday, June 5 4:30 PM - 5:45 PM, S210 E:
DAT312 - XML in Microsoft SQL Server 2005
Speaker(s): Peter DeBetta
XML is now a native type in SQL Server 2005. This session explores this new type, XML methods, new features for shredding and composing XML, and provides an introduction to XQuery, XPath, and some preferred practices for implementing XML, and other XML-related topics.

Wednesday, June 6 3:45 PM - 5:00 PM, Blue Theater 12:
DAT18-TLC - Overview of New T-SQL Programmability Features in the Next Release of Microsoft SQL Server
Speaker(s): Michael Wang
This session offers a high level overview of all new T-SQL programmability features introduced in next release of SQL Server in terms of SQL type system, DDL, server procedure programming, supportability, and globalization.
Friday, June 8 10:45 AM - 12:00 PM, Blue Theater 12:
DAT06-TLC - New T-SQL System Types in Microsoft SQL Server Codename "Katmai"
Speaker(s): Michael Wang
This session focuses on all new T-SQL system data types introduced in next release of SQL Server.

I hope to see you there or at the TLC booth of SQL Server.

Now that we can publicly talk about "Katmai", I will be blogging more about what's new and be available to take your feedback.