using System;
using System.Collections.Generic;
using System.Text;
using System.IO;
using System.Net;
using System.Net.Sockets;

namespace SymbolFilter
{
    class FilterProxy
    {
        // we'll accept connections using this listener
        static TcpListener listener;

        // this holds the white list of DLLs we do not ignore
        static List dllFilterList = new List();

        // default port is 8080, config would be nice...
        const int port = 8080;

        static void Main(string[] args)
        {
            // load up the dlls
            InitializeDllFilters();

            // open the socket
            StartHttpListener();

            // all real work happens in the background, if you ever press enter we just exit
            Console.WriteLine("Listening on port {0}.  Press enter to exit", port);
            Console.ReadLine();
        }

        static void InitializeDllFilters()
        {
            try
            {
                // we're just going to throw if it fails...
                StreamReader sr = new StreamReader("symfilter.txt");

                // read lines from the file
                string line;
                while ((line = sr.ReadLine()) != null)
                {
                    dllFilterList.Add(line.Trim().ToLowerInvariant());
                }

                sr.Close();

            }
            catch (Exception e)
            {
                // anything goes wrong and we're done here, there's no recovery from this
                Console.WriteLine(e.Message);
                Environment.Exit(1);
            }
        }

        // Here we will just listen for connections on the loopback adapter, port 8080
        // this could really use some configuration options as well.  In fact running more than one of these
        // listening to different ports with different white lists would be very useful.
        public static void StartHttpListener()
        {
            try
            {
                listener = new TcpListener(IPAddress.Loopback, port);
                listener.Start();
                listener.BeginAcceptTcpClient(new AsyncCallback(DoAcceptTcpClientCallback), listener);
            }
            catch (Exception e)
            {
                // anything goes wrong and we're done here, there's no recovery from this
                Console.WriteLine(e.Message);
                Environment.Exit(1);
            }

        }

        // when we accept a new listener this callback will do the job
        public static void DoAcceptTcpClientCallback(IAsyncResult ar)
        {
            // Get the listener that handles the client request.
            TcpListener listener = (TcpListener)ar.AsyncState;

            // I should probably assert that the listener is the listener that I think it is here

            // End the operation and display the received data on  
            // the console.
            try
            {
                // end the async activity and open the client
                // I don't support keepalive so "using" is fine
                using (TcpClient client = listener.EndAcceptTcpClient(ar))
                {
                    // Process the http request
                    // Note that by doing it here we are effectively serializing the listens
                    // that seems to be ok because mostly we only redirect
                    ProcessHttpRequest(client);
                }
            }
            catch (Exception e)
            {
                // if anything goes wrong we'll just move on to the next connection
                Console.WriteLine(e.Message);
            }

            // queue up another listen
            listener.BeginAcceptTcpClient(new AsyncCallback(DoAcceptTcpClientCallback), listener);
        }


        // we have an incoming request, let's handle it
        static void ProcessHttpRequest(TcpClient client)
        {
            // we're going to process the request as text
            NetworkStream stream = client.GetStream();
            StreamReader sr = new StreamReader(stream);

            // read until the first blank line or the end, whichever comes first
            var lines = new List();
            for (;;)
            {
                var line = sr.ReadLine();

                if (line == null || line == "")
                    break;

                lines.Add(line);
            }

            // e.g. "GET /foo.pdb/DE1EBC3EE7E542EA96B066229D3A40081/foo.pdb HTTP/1.1"
            var req = lines[0];

            // avoid case sensitivity issues for matching the pattern
            var reqLower = Uri.UnescapeDataString(req).ToLowerInvariant();

            // loop over available patterns, if any matches early out
            int i;
            for (i = 0; i < dllFilterList.Count; i++)
            {
                if (reqLower.Contains(dllFilterList[i]))
                    break;
            }

            // if we didn't match, or it isn't a GET or it isn't HTTP/1.1 then serve up a 404
            if (i == dllFilterList.Count || !req.StartsWith("GET /") || !req.EndsWith(" HTTP/1.1"))
            {
                // you don't match, fast exit, this is basically the whole point of this thing
                Return404(client);
            }
            else
            {
                // this is the real work
                Console.WriteLine("Matched pattern: {0}", dllFilterList[i]);
                RedirectRequest(client, req);
            }
        }

        // cons up a minimal 404 error and return it
        static void Return404(TcpClient client)
        {
            // it doesn't get any simpler than this
            var sw = new StreamWriter(client.GetStream());
            sw.WriteLine("HTTP/1.1 404 Not Found");
            sw.WriteLine();
            sw.Flush();
        }

        // cons up a minimal 404 error and return it
        static void Return302(TcpClient client, string server, string url)
        {
            string line = String.Format("Location: http://{0}{1}", server, url);
            Console.WriteLine("302 Redirect {0}", line);

            // emit the redirect
            var sw = new StreamWriter(client.GetStream());
            sw.WriteLine("HTTP/1.1 302 Redirect");
            sw.WriteLine(line);
            sw.WriteLine();
            sw.Flush();
        }


        static void RedirectRequest(TcpClient client, string req)
        {
            // we know this is safe to do because we already verified that the request starts with "GET /"
            string request = req.Substring(5); // strip off the "GET /"
            request = request.Substring(0, request.Length - 9);  // strip off " HTTP/1.1"

            // we're looking for the server that we are supposed to use for this request in our path
            // the point of this is to make it so that you can set your symbol path to include
            // http://localhost:8080/your-sym-server/whatever
            // and your-sym-server will be used by the proxy
            int delim = request.IndexOf('/', 1);

            // if there is no server specified then we're done
            if (delim < 0)
            {
                // serve up an error
                Return404(client);
                return;
            }

            // the target server is everything up to the / but not including it
            string targetServer = request.Substring(0, delim);

            // the new request URL starts with the first character after the |
            request = request.Substring(delim + 1);

            // if there isn't already a leading slash, then add one
            if (!request.StartsWith("/"))
                request = "/" + request;

            Return302(client, targetServer, request);
        }
    }
}

They are here and here.

Some people thought I was nuts.

Some people provided benchmarks "proving" that 64 bit code is in fact faster than 32 bit code.

I guess the lesson for me here is that I can't say often enough that I only ever write approximately correct blog entries and YMMV should be stamped on top of everything.

But, just so you don't think I've lost my mind, here is some data.

These are recent builds of IE and Edge in 32 bit and 64 bit.  Its the same code, or as close as possible to the same code.  This is just the main engine of both plus the old script engine and the new script engine.

This result is completely typical -- between 1.2 and 1.4x growth is very normal for mainstream code.  This is not universal, but it's in line with what I've seen elsewhere.

Yes there are exceptions.  Yes it totally depends on the codebase.

Now is this going to be slower?  Almost certainly because bigger is slower.

Why almost?

Well, if the code has really good locality (e.g. you only use a tiny slice of it;  e.g. you move from one well used slice to another well used slice)  then it might be that the parts of the code that are hot at any given moment are still fitting well into the cache.  But that's sort of a lie as well.  You see in a real system there is pollution because device drivers are running and background processes are running and so it's never really the case that there is surplus cache capacity.  In real systems extra cache capacity basically just creates the opportunity for your code to remain efficient in the presence of other normal workloads.  Micro-benchmarks don't these effects.

I also said that the data is always bigger.  People disagree, but there really is no room for disagreement here.  I think we can all agree that going to 64 bits won't make your data smaller, so the best you can hope for is a tie.  I also think you are going to have at least one pointer on the stack somewhere.   So a tie isn't possible.  Pedantic yes, but on point.  The truth is that depending on your workload you will see varying amounts of growth and no shrinkage.  If your growth is sufficiently small, or the locality of the data is sufficiently good, then the extra cache misses from size growth will not be so bad.  Micro-benchmarks will often still fit in the cache on a 64 bit system. 

Visual Studio is in a poor position viz data growth because it has a lot of pointer rich data structures.  Microsoft Edge, Chrome, and Firefox are in relatively better positions because much of their data is not rich in pointers -- bitmaps for instance are essentially (minus some metadata) identical, strings are identical, styles can be stored densely.  As a consequence browsers will suffer less than VS would.

The bottom line is that this will vary a great deal based on your workload.  But the price of cache misses should not be underestimated.  A modern processor might retire over 200 instructions in the same time as one cache miss.  More in extreme situations.  That's a lot to recover from, extra registers won't do it universally.

I wrote about these phenomena in greater detail in this article from 2014.  That article includes a quantitative examination of data growth and locality effects.

I had some top level observations for anyone contemplating a port:

* if you think you're going to get free speed, you have it backwards -- you start in the hole, you will have to win back your losses with processor features, definitely possible but not a given

* if you're going to 64 bits because you're out of memory but you could fix your memory problems with "let's not do this stupid thing anymore" then you should do that

* if you're avoiding 64 bits because you keep saying "I can stay in 32 bits if I do this new stupid thing" then you're kidding yourself, crazy workarounds are doomed

Visual Studio in 2009 was definitely in the situation whereby avoiding some dumb stuff we could make it fit.  I don't know where it is in 2016.

This also excludes collateral benefits you get from going to 64 bits, such as avoiding the wow subsystem, cleaning up an otherwise potentially clumsy architecture, greater security due to increased exploit difficulty.  Some of these benefits may be very important to your workload/customers. 

[I don't have to go out on a limb to acknowledge that is the worst article I've ever written.  I wrote it in the wee hours one morning in a rotten mood and it shows.  There are far too many absolutes that should have been qualified and the writing style is too aggressive for no good reason.  I'm not taking it down because there are worthy comments, and I refuse to try to pretend it never happened.  But I absolutely regret writing this article in this way.  If you choose to read this, use a large sanity filter and look at some of the comments and the follow-up for qualifications to help see what I'm getting at.]

[This article is in response to an earlier posting which you can find here]

I’ve been reading some of the commentary on my post about 64-bit Visual Studio which is really about 64-bit vs. 32-bit generally using Visual Studio as an example and I have to say that for the most part, I’m pretty disappointed with the arguments being put forth in favor of 64-bits.

[Some less than charitable and totally unnecessary text removed.  I blame myself for writing this at 2:30am.  It was supposed to be humorous but it wasn't.]

There is an argument to be made here, but there is also a great deal of ignoring of the real issue going on here.

Let’s actually go about doing the job of properly attacking my position the way I think it should be attacked, shall we?

I start with some incontrovertible facts.  Don’t waste your time trying to refute them, you can’t refute facts. You can have your own opinion, but can’t have your own facts.

The relevant facts are these:

-the same algorithm coded in 64-bits is bigger than it would be coded in 32-bits

-the same data coded for 64-bits is bigger than it would be coded in 32-bits

-when you run the same code, but bigger encoding, over the same data, but bigger encoding, on the same processor, things go slower

-any work I can possibly do has an opportunity cost which will mean there is some other work I can’t do

All righty, it’s hard to argue with those.

Now let’s talk about the basis I use for evaluation.

-I get points for creating a great customer experience

-I get no points for using technology X, only for the experience, using fewer technologies for the same experience is better than using more

-I get no points for using more memory, not even enabling the use of more memory, only for the experience, using less memory for the same experience is better than using more

OK, so in short, I begin with “64-bits gets no free inherent value, it has to justify itself with Actual Benefits like everything else.”

We cannot make a compelling argument with fallacies like “32 bits was better than 16 therefore 64 must be better than 32”, nor will we get anywhere with “you’re obviously a short-sighted moron.”

But maybe there is something to learn from the past, and what’s happened over the last 6 years since I first started writing about this.

For Visual Studio in particular, it has been the case since ~2008 that you could create VS extensions that were 64-bits and integrate them into VS such that your extension could use as much memory as it wanted to (Multi-process, hybrid-process VS has been a thing for a long time).  You would think that would silence any objections right there -- anyone who benefits from 64-bits can be 64-bits and anyone who doesn’t need 64-bits can stay 32-bits.  It’s perfect, right?

Well, actually things are subtler than that.

I could try to make the case that the fact that there are so few 64-bit extensions to VS is proof positive that they just aren’t needed.  After all, it’s been nearly 8 years, there should be an abundance of them.  There isn’t an abundance, so, obviously, they’re not that good, because capitalism.

Well, actually, I think that argument has it exactly backwards, and leads to the undoing of the points I made in the first place.

The argument is that perhaps it’s just too darn hard to write the hybrid extensions.  And likewise, perhaps it’s too darn hard to write “good” extensions in 32-bits that use memory smartly and page mostly from the disk.  Or maybe not even hard but let’s say inefficient –from either an opportunity cost perspective or from a processor efficiency perspective; and here an analogy to the 16-bit to 32-bit transition might prove useful.

It was certainly the case that with a big disk and swappable memory sections any program you could write in 32-bit addressing could have been created in 16-bit (especially that crazy x86 segment stuff).  But would you get good code if you did so?  And would you experience extraordinary engineering costs doing so?  Were you basically fighting your hardware most of the time trying to get it to do meaningful stuff?  It was certainly that case that people came up with really cool ways to solve some problems very economically because they had memory pressure and economic motivation to do so.  Those were great inventions.  But at some point it got kind of crazy.  The kind of 16-bit code you had to write to get the job done was just plain ugly.

And here’s where my assumptions break down.  In those cases, it’s *not* the same code.  The 16-bit code was slow ugly crapola working around memory limits in horrible ways and the 32-bit code was nice and clean and directly did what it needed to do with a superior algorithm.  Because of this, the observation that the same code runs slower when it’s encoded bigger was irrelevant.  It wasn’t the same code!  And we all know that a superior algorithm that uses more memory can (and often does) outperform an inferior algorithm that’s more economical in terms of memory or code size.

Do we have a dearth of 64-bit extensions because it’s too hard to write them in the hybrid model?

Would we actually gain performance because we wouldn’t have to waste time writing tricky algorithms to squeeze every byte into our 4G address space?

I don’t have the answer to those questions.  In 2009 my thinking was that for the foreseeable future, the opportunity cost of going to 64-bits was too high compared to the inherent benefits.   Now it’s 2016, not quite 7 years since I first came to that conclusion.  Is that still the case?

Even in 2009 I wanted to start investing in creating a portable 64-bit shell* for VS because I figured the costs would tip at some point. 

I don’t work on Visual Studio now, I don’t know what they’re thinking about all this.

If there’s a reason to make the change now, I think I’ve outlined it above. 

What I can say is that even in 2016, the choice doesn’t look obvious to me.   The case for economy is still strong.  And few extensions are doing unnatural things because of their instruction set – smart/economical use of memory is not unnatural.  It’s just smart.

*the "Shell" is the name we give to the core of VS (what you get with no extensions, which is nearly nothing, plus those few extensions that are so indispensable that you can't even call it VS if you don't have them, like solutions support -- that's an extension]

You just can’t help but fall in the pit for the win.

I like this metaphor because it’s provocative and it’s geometric.  It’s about changing the rules so that people do the right things by default, and so, win in three easy steps:

1. Use framework
2. Do obvious/easy/natural things
3. Profit

I think about many things this way, especially longer term things, like affecting behavior in an organization.

Imagine that systems of your organization are like a complicated piece of terrain.  The people operating in the system are little balls rolling around.  Victory is some sweet spot in the terrain somewhere.   Depending on where Victory is it might be pretty tricky to get there.  Maybe it’s up some big hill.  Maybe there’s a lot of potholes.  Maybe the path isn’t very direct.

If you had a Pit of Success organization, then Victory would literally be this GIANT hole in the middle that you just automatically fall into.  You can’t help but win.

Anyone work at a place like that? 

Me either.

Here’s the kicker.  The terrain tends to get worse rather than better as things get complicated.  For pretty simple reasons too.  For instance, new parts of the system tend to be based on the old parts.  So, say you have part of the terrain that’s full of potholes.  Yup, naturally when adding to the system people are going to copy them, making EVEN MORE potholes for people to fall into.

Soon you get a situation where making any kind of progress requires complicated manuals and so forth.  Step to the left.  Avoid the spikes, don’t break your ankles on any of those holes, roll uphill over two hills, don’t crash into any obstacles, and maybe if you’re lucky you can claim victory.

Now you can either try to make your organization more effective by teaching everyone the rules, and I guess that doesn’t hurt, some rules are probably pretty good, or you can go about the business of making the terrain better.  Change the shape of some of those hills so that people tend to roll in the right direction.  Move the victory circle somewhere easy to get to.   And OMG fill in those potholes so people stop cloning them!

If you do these things, then over time you will tend to get more people automatically going down good paths to success.  They might not even know you did it, they just suddenly start winning doing the obvious “downhill” stuff.

I’ll leave you with my favorite Pit of Success metaphor.  It’s an apocryphal story about the hamburger recipes of Dean Martin and Frank Sinatra.

It seems Mr. Martin was famous for his excellent burgers and had encoded a fabulous way to prepare them.  There were many important instructions which I don’t remember but stuff like getting only the best meat, and getting the right spices, and cooking time, and of course the perfect wine. 

Mr. Sinatra’s recipe on the other hand goes like this:

1. Go to Deano’s house
2. Ask Deano to make you a hamburger
3. Drink Deono’s wine.

I’m liking that recipe :)

Now if you’re a framework author, or an organizational architect then maybe you’ve gotta be more like Dean but it’s be really nice if pretty much everyone else could be Frank.

Shameless plug:  please have a look at my words on Women in Computer Science if you haven't already

Today I have something to ask.

I know my subscribers include tons of competent software engineers. I have a request/challenge for you. Especially the men.

Will any of you stand with me?

All you have to do is this:

1) Do something for a school, any school that means something to you

2) Seek people to mentor, my focus is on women, but you can have any focus that means something to you

3) Tell your friends this is something you stand for

That's all. It's been one of the most rewarding things I've ever done and I wish it for all of you.

‪#‎heforshe

I don't usually put my name on things, especially charitable giving, but in this case it was important to me to not only give the money but to actually come out and say "This is something I stand for."

A few years ago (ok 10) I was profoundly affected by one of the Microsoft Think Week Papers I was asked to referee and it changed the way that I thought about women in CS. It showed me that there were things I could personally do, right away, that would make a difference, and so I set about doing them. I wrote about some of these things a few days ago when I discussed Privilege.

I'm putting my name on this ‪#‎heforshe‬ effort because I hope to inspire others to do the same (it's super easy) and because in addition to the money, I want to give the winners some kind of leg up. An industry contact could be invaluable to a person trying to break into the field. If I'm lucky, some of them will come to trust me for advice. If I'm very lucky I'll make a difference for at least some of them. At worst, there will be a few people in the world with smaller student loans.

This is just one small thing, but if many of us help a little in this way, we can do something really meaningful.

[Edit: naturally I *cough* totally planned that this would happen during Computer Science Week  -- serendipity FTW]

See Part 2