Solution 1: Change the Recovery Model to Simple

This would make sure that the transaction logs aren't included for backup, thereby eliminating the error.

Solution 2: Backup the log before running staging service

Execute the SQL command BACKUP LOG ECommOnline_productcatalog TO DISK='c:\temp.log' and get the log backed up. Restart the Staging service. Also, you can include this as a scheduled maintenance so that the backup is performed just before staging is invoked.

So the first thing I did was to look up the Internet and found this wadewegner blog entry which perfectly addressed my issue. To make sure that the entry doesn't get lost somewhere, I am reproducing it:

On the other hand, you might also want the user to be forced off the Secure protocol for general view pages so that the network bottleneck is eliminated at the server end due to unwanted overuse of HTTPS. One of the best ways to achieve this is using HttpModules in ASP.NET which provides a very powerful mechanism to intercept HTTP requests and redirect them as necessary.

To effectively develop HttpModule you need to

1. Hook up the module during the OnInit event

2. Trap the request during the PreRequestHandler event.

Digging into the code, it would be something like this:

public class SslSwitchModule : IHttpModule
    {
        //store your secure pages in a hastable for fast retrieval.
        //this can be populated when the application starts up so that repeated 
        // overhead is avoided.
        private static Hashtable securePages = null;
        
        
        public void ProcessRequest(HttpContext context)
        {
            Uri requestUri = context.Request.Url;

            //if the request is for HTTP, check if HTTPS is needed
            if (!context.Request.IsSecureConnection)
            {
                string urlRequested =
 HttpUtility.UrlDecode(context.Request.Path.ToUpper().Replace(context.Request.ApplicationPath.ToUpper(),
""));
                if (SecurePages.ContainsValue(urlRequested))
                {
                    //switch to HTTPS
                    string secureUrl = "https" + context.Request.Url.AbsoluteUri.Substring(4);
                    context.Response.Redirect(secureUrl, true);
                }
            }
            else
            {
                //if the url requested is inside the https,
                // determine if its needed to be in that page
                string urlRequested =
 HttpUtility.UrlDecode(context.Request.Path.ToUpper().Replace(context.Request.ApplicationPath.ToUpper(),
 ""));
                if (!SecurePages.ContainsValue(urlRequested))
                {
                    //switch to HTTPS
                    string unSecureUrl = "http" + context.Request.Url.AbsoluteUri.Substring(5);
                    context.Response.Redirect(unSecureUrl, true);
                }
            }
   
        }        
      

        #region IHttpModule Members

        public void Dispose()
        {
            ;
        }

        public void Init(HttpApplication context)
        {   
             // wireup the event for processing
             context.PreRequestHandlerExecute += new EventHandler(context_PreRequestHandlerExecute);
            
        }

        void context_PreRequestHandlerExecute(object sender, EventArgs e)
        {
            HttpApplication httpApp = (HttpApplication)sender;
            //process the request
            this.ProcessRequest(httpApp.Context);
        }
       
        #endregion
    }

A frequently searched topic is to create, save and access Multi-Valued properties for the UserObject in the profile system.

1. Create Multi-valued profile properties
This is a straightforward profile editing activity on the commerce server, so would not go into it!

2. Saving into Multi-valued property
The below code snippet would do the trick for you. The key here is to set the new property value into an array and then assign the array to the Commerce Server Property.

//create a GUID if not present
         emailToAdd.Id = Guid.NewGuid();
         //create the custom profile
         Profile emailProfile =
                 commerceContext.CreateProfile(emailToAdd.IdString, 
CommerceConstants.SAVED_EMAIL_PROFILE);
         //save properties first
         emailProfile.Properties[SavedEmailProfilePropertyNames.EmailId].Value 
             = emailToAdd.IdString;
         emailProfile.Properties[SavedEmailProfilePropertyNames.EmailName].Value 
             = emailToAdd.Name;
         //update back to the store
         emailProfile.Update();
         //refresh the cache
         emailProfile.Refresh();
         
         //get the owner profle
         using (Profile custProfile =
            commerceContext.GetProfile(CustProfilePropertyNames.Id, this.IdString,
            CommerceConstants.CUST_PROFILE))
         {              
             //if found, proceed for update
             if (custProfile != null)
             {

                 string[] additionalData;
                 //get preexisting multi values
                 object[] savedEmails = 
(object[])custProfile.Properties[CustProfilePropertyNames.ContactEmails].Value;
                 if (savedEmails != null)
                 {
                     //expand by 1
                     additionalData = new string[savedEmails.Length + 1];
                     int i = 0;
                     //assign back the current data to a new array
                     foreach (object item in savedEmails)
                     {

                         if (item != null && !(item as string).Equals("NULL"))
                         {
                             additionalData[i++] = item as string;
                         }
                     }
                     //add the item
                     additionalData[i] = emailToAdd.IdString;

                 }
                 else
                 {
                     //if no items preexist, add
                     additionalData = new string[1];
                     additionalData[0] = emailToAdd.IdString;
                 }
                 //save the multi valued items
                 custProfile.Properties[CustProfilePropertyNames.ContactEmails].Value = 
additionalData;
                 //update
                 custProfile.Update();
                 //refresh
                 custProfile.Refresh();
                 
             }
             else
             {
                 //no profile found, something not right!
                 throw new Exception("No profile found for user");
             }
         }
     }

3. Modifying Multi-valued property

The modifying the multi valued property is quite straight forward. If you are storing the profile Ids as done in the above example, you need to update the real/child profile rather than the multi-valued property. If you are directly consuming the multi-valued property, then you need to pass on the previous value, get the list of values as shown above, modify the required one and assign back the array and update.

4. Deleting Multi-valued property

Again, on the lines above, if you are using profile Ids, then delete the child profile, remove the Id from the array and save it back. If you are using the property itself, skip the item when assigning to the temporary array and save it back. One "GOTCHA" is when the multi-valued property has only one element - in this case use DBNull.Value to assign:

custProfile.Properties[CustProfilePropertyNames.ContactEmails].Value = DBNull.Value;

Case in point is the exceptionally well written Data Access Application Block (a.k.a. DAAB). My team is involved in developing an e-commerce web site for a large automotive retailer and is using this DAAB beauty to ease the development efforts. We integrated the default provided DAAB DLLs into the solution and happily developed the data layers for our system. It was working just fine on each of the dev machines;  I put the solution to a production server so that our clients can appreciate the beauty of it! The next thing I know, the site starts timing out for some of the calls. I went into the logs and found that the connection timeout errors were rampant! I started digging in to the code and found that we had used one particular method a lot and this was the only method that did not close the connection "automatically" - the ExecuteReader method.

If you think about it logically, it makes a lot of sense NOT to close the connection automatically as the DAAB does not have any idea as to when the reader will move out of scope. So the correct code snippet to use is

   1:  Database db = DatabaseFactory.CreateDatabase();

   2:   

   3:  DbCommand dbCommand = db.GetSqlStringCommand("Select Name, Address From Customers");

   4:  using (IDataReader dataReader = db.ExecuteReader(dbCommand))

   5:  {

   6:      // Process results

   7:  } 

The full reasoning is well explained at the MSDN Site here

The import failed because the import file is larger than the maximum size allowed on your server.  Contact your system administrator.

The solution for this to open up the web.config of CatalogWebService and then increase the size limit of uploads to more than the file that you intend to upload. The current default value is 200MB. The steps to increase the size are:

1. Traverse to the path which has the Catalog Web Service (typically C:\Inetpub\wwwroot\CatalogWebService )
2. Open up the web.config
3. Set the value of maxUploadFileSize attribute to the value which you need

Now, you should be able to upload the import the file.

The situation gets even more complicated when you have certain documents that can be downloaded by everyone on the Internet and have some documents visible only to logged in users. A HttpHandler fits this scenario perfectly.

Before I get into developing the HttpHandler, a couple of points regarding the design is a must:

• In case you need to distinguish between allowing the users to download publicly visible files and those that need log-in, its a good idea to have the documents in different folders so that security can be enforced more strictly.
• For documents that need user's authentication and / or authorization, its advisable to have the document store outside the Web's root to make sure that the folder is not visible to any automated download app.
• In cases wherein there is a 1-1 map between the users and his / her documents, a unique way to determine the user's file path needs to be put it to tighten security. This can be in the form of a HashedId for the user which maps onto the path in the store or some other URL rewriting technique.

To get on with it, there are three major steps that need to be performed to make sure that the file request is routed through the IIS via the ASPNET pipeline and onto your handler.

1. Develop your handler - of course this is obvious :-)
2. Configure your web site to use this handler for the filetype that you want authorization on and the verbs that you'll need to support (GET , PUT, POST, etc)
3. Configure the IIS to make sure that the request for the particular filetype is routed through the ASPNET ISAPI dll so that your custom handler is it.

Lets take it step by step...and to make it more fun, in the reverse order!

Step 3: Configure IIS

This is quite a straightforward step. All you need to do is open up the properties of the Web Application under consideration.

1. Under the Virtual Directory tab,
2. click on the Configuration... button,
3. in the new window that opens up, under the Mappings tab,
4. click Add...,
5. click Browse and select C:\<windows root> \Microsoft.NET\Framework\<version>\aspnet_isapi.dll (the full path looks something like: C:\WINNT\Microsoft.NET\Framework\v2.0.50727\aspnet_isapi.dll).
6. Provide the Extension without the dot (i.e just PDF)
7. Under the Verbs section, you can leave it as is or for better security, provide only GET.
8. Uncheck the Verify File Exists checkbox.
9. Click OK, Apply and finally make sure that you have Execute Permissions to Scripts Only.

Step 2: Configure your Web Application

All you need to do is add the below line(s) into the web.config of the application you are developing:

<system.web>
    <httpHandlers>
        <add verb="GET" path="*.pdf" 
             type="Microsoft.Web.PortalFramework.HttpHandlers.PdfDownloadAuthorizationHandler,
             Microsoft.Web.PortalFramework" />
    </httpHandlers>
</system.web>

The above states that the HTTP verb GET with all files ending in extenion pdf should be routed to the class mentioned in the type. The type if of format "<Namespace.Classname>, <AssemblyName>"

Step 1: Develop your Handler

All you need is for your class to implement IHttpHandler interface in the System.Web namespace and that's its! Instead of saying some blah blah blah about the code, here's the full code:

using System;
using System.IO;
using System.Collections.Generic;
using System.Text;
using System.Web;


namespace Microsoft.Web.PortalFramework.HttpHandlers
{
    /// <summary>
    /// 
    /// </summary>
    public class PdfDownloadAuthorizationHandler: IHttpHandler
    {
        private static string userDocsBaseFolder;
        /// <summary>
        /// Initializes a new instance of the <see cref="DownloadAuthorizationModule"/> class.
        /// </summary>
        public PdfDownloadAuthorizationHandler()
        {
            userDocsBaseFolder = "PersonalDocs";
        }
       

        #region IHttpHandler Members

        /// <summary>
        /// Gets a value indicating whether another request can 
        /// use the <see cref="T:System.Web.IHttpHandler"></see> instance.
        /// </summary>
        /// <value></value>
        /// <returns>true if the <see cref="T:System.Web.IHttpHandler"></see>
        /// instance is reusable; otherwise, false.</returns>
        public bool IsReusable
        {
            get { return false; }
        }

        /// <summary>
        /// Enables processing of HTTP Web requests by a custom HttpHandler
        /// that implements the <see cref="T:System.Web.IHttpHandler"></see> interface.
        /// </summary>
        /// <param name="context">An <see cref="T:System.Web.HttpContext"></see>
        /// object that provides references to the intrinsic server objects 
        /// (for example, Request, Response, Session, and Server) 
        /// used to service HTTP requests.</param>
        public void ProcessRequest(HttpContext context)
        {
            try
            {
                //get the url requested by the user
                string urlRequested = 
                    HttpUtility.UrlDecode(context.Request.Url.AbsolutePath.ToUpper());
                //set the initial flag to false
                bool userPersonalDoc = false;
                //remove all spaces in the url before checking if the url contains the private path
                if (urlRequested.Replace(" ", "").Contains(userDocsBaseFolder))
                {
                    //once we know that its for the personal folders, set the flag
                    userPersonalDoc = true;
                    //we now assume that the cookie is set with the document profile id of the user
                    //and the dynamic link that its generated includes this key
                    //check only for the negative conditions
                    HttpCookie profCookie = context.Request.Cookies["DocId"];
                    if (profCookie != null)
                    {
                        //we need to use some sort of encryption (refer my previous post!)
                        TripleDESCryptoHelper helper = new TripleDESCryptoHelper();
                        string profileId = helper.GetDecryptedValue(profCookie.Value);
                        Guid profileGuid = new Guid(profileId);
                        //check if the docid from the url and the one in the cookie match
                        if (!urlRequested.Contains(profileGuid.ToString("B").ToUpper()))
                        {
                            //no match - spoofed request, send 'em back
                            context.Response.Redirect(context.Request.ApplicationPath,true);
                            
                        }
                    }
                    else
                    {                        
                        context.Response.Redirect(context.Request.ApplicationPath);
                    }
                }
                //if we have reached till here, we are good to go
                this.TransmitRequestedFile(context, urlRequested, userPersonalDoc);
            }
            catch (Exception generalException)
            {
                ExceptionManager.LogException(generalException);
            }


        }


        /// <summary>
        /// Transmits the requested file.
        /// </summary>
        /// <param name="context">The context.</param>
        /// <param name="urlRequested">The URL requested.</param>
        /// <param name="forceDownload">if set to <c>true</c> [force download].</param>
        private void TransmitRequestedFile(HttpContext context,
            string urlRequested, bool forceDownload)
        {
            // as a security considerataion, force the user to download the document
            // if the doc is from the personal folder
            if (forceDownload)
            {
                this.StartForcedDownload(context, urlRequested);
            }
            else {
                //else let the IE handle the filetype
                this.TransmitNormal(context, urlRequested);
            }
        }

        /// <summary>
        /// Transmits the file in 'normal' way.
        /// </summary>
        /// <param name="context">The context.</param>
        /// <param name="urlRequested">The URL requested.</param>
        private void TransmitNormal(HttpContext context, string urlRequested)
        {
            string filePath = HttpContext.Current.Server.MapPath(urlRequested);

            string fileName = System.IO.Path.GetFileName(filePath);

            context.Response.ClearContent();

            context.Response.ClearHeaders();            

            FileInfo pdfInfo = new FileInfo(filePath);

            context.Response.AddHeader("Content-Length", pdfInfo.Length.ToString());

            //assuming that the file is pdf, needs to be changed appropriately
            //context.Response.ContentType = "application/zip";
            context.Response.ContentType = "application/pdf";

            context.Response.TransmitFile(filePath);

            context.Response.End();
        }

        /// <summary>
        /// Starts the forced download.
        /// </summary>
        /// <param name="context">The context.</param>
        /// <param name="urlRequested">The URL requested.</param>
        private void StartForcedDownload(HttpContext context, string urlRequested)
        {
            string filePath = HttpContext.Current.Server.MapPath(urlRequested);

            string fileName = System.IO.Path.GetFileName(filePath);

            context.Response.ClearContent();

            context.Response.ClearHeaders();

            context.Response.AddHeader("Content-Disposition", "inline; filename=" + fileName);

            context.Response.ContentType = "application/pdf";

            context.Response.BufferOutput = false;

            context.Response.TransmitFile(filePath);
                        
            context.Response.End();

        }

        #endregion
    }
}

Though the CryptoSampleCSSample.msi provides some samples as to how to achieve this. It fails to address one of the most common scenarios of storing the IV (the Initialization Vector) and the Key in an app.config (for Windows / Console Applications) and in web.config (for Web Applications).

This complete code demonstrates how to pickup IV and Key from the web.config / app.config and then use the values in encrypting and decrypting text values. The key thing to note is that the string needs to be converted using:

Convert.FromBase64String(IV)

NOTE: The IV and Key can be generated by using the CryptoSampleCS.exe provided in the MSDN Article mentioned above.

First, the app.config:

<?xml version="1.0"?>

<configuration>
    <appSettings>
        <add key="IV" value="SuFjcEmp/TE="/>
        <add key="Key" value="KIPSToILGp6fl+3gXJvMsN4IajizYBBT"/>
    </appSettings>
</configuration>

Now the full code: 

using System;
using System.Collections.Generic;
using System.Text;
using System.Security.Cryptography;
using System.IO;
using System.Configuration;

    public class CryptoHelper
    {

        //private readonly string IV = "SuFjcEmp/TE=";
        private readonly string IV = string.Empty;
        //private readonly string Key = "KIPSToILGp6fl+3gXJvMsN4IajizYBBT";
        private readonly string Key = string.Empty;

        /// <summary>
        /// Initializes a new instance of the <see cref="CryptoHelper"/> class.
        /// </summary>
        public CryptoHelper()
        {
            IV = ConfigurationManager.AppSettings["IV"];
            Key = ConfigurationManager.AppSettings["Key"];
        }

        /// <summary>
        /// Gets the encrypted value.
        /// </summary>
        /// <param name="inputValue">The input value.</param>
        /// <returns></returns>
        public string GetEncryptedValue(string inputValue)
        {
            TripleDESCryptoServiceProvider provider = this.GetCryptoProvider();
            // Create a MemoryStream.
            MemoryStream mStream = new MemoryStream();

            // Create a CryptoStream using the MemoryStream 
            // and the passed key and initialization vector (IV).
            CryptoStream cStream = new CryptoStream(mStream,
                provider.CreateEncryptor(),CryptoStreamMode.Write);

            // Convert the passed string to a byte array.: Bug fixed, see update below!
            // byte[] toEncrypt = new ASCIIEncoding().GetBytes(inputValue);
            byte[] toEncrypt = new UTF8Encoding().GetBytes(inputValue);

            // Write the byte array to the crypto stream and flush it.
            cStream.Write(toEncrypt, 0, toEncrypt.Length);
            cStream.FlushFinalBlock();

            // Get an array of bytes from the 
            // MemoryStream that holds the 
            // encrypted data.
            byte[] ret = mStream.ToArray();

            // Close the streams.
            cStream.Close();
            mStream.Close();

            // Return the encrypted buffer.
            return Convert.ToBase64String(ret);
        }

        /// <summary>
        /// Gets the crypto provider.
        /// </summary>
        /// <returns></returns>
        private TripleDESCryptoServiceProvider GetCryptoProvider()
        {
            TripleDESCryptoServiceProvider provider = new TripleDESCryptoServiceProvider();
            provider.IV = Convert.FromBase64String(IV);
            provider.Key = Convert.FromBase64String(Key);
            return provider;            
        }

        /// <summary>
        /// Gets the decrypted value.
        /// </summary>
        /// <param name="inputValue">The input value.</param>
        /// <returns></returns>
        public string GetDecryptedValue(string inputValue)
        {
            TripleDESCryptoServiceProvider provider = this.GetCryptoProvider();
            byte[] inputEquivalent = Convert.FromBase64String(inputValue);
            // Create a new MemoryStream.
            MemoryStream msDecrypt = new MemoryStream();

            // Create a CryptoStream using the MemoryStream 
            // and the passed key and initialization vector (IV).
            CryptoStream csDecrypt = new CryptoStream(msDecrypt,
                provider.CreateDecryptor(),
                CryptoStreamMode.Write);
            csDecrypt.Write(inputEquivalent, 0, inputEquivalent.Length);
            csDecrypt.FlushFinalBlock();

            csDecrypt.Close();

            //Convert the buffer into a string and return it.
            return new UTF8Encoding().GetString(msDecrypt.ToArray());
        }
    }

 Update: JT Carvalho emailed me about a bug in the code above.
"You are encoding the crypted value with AsciiEncoding and decoding it with UTF8Encoding, so in some special chars (like portuguese chars) they will not match.
I think encoding it with UTF8Encoding it will resolve this."

Thanks for pointing this out and thanks for the fix as well!

I will try to get hold of some genuine code samples that are spread across the MSDN blogs including the blogs of key developers who worked on the product. I will be posting more such items as and when I find them (or I find time to find them). I'll also be including code snippets of some real-life searches using the Commerce Server 2007 APIs that am working on.

A terrific set of code samples for the Commerce Server 2007 Catalog System has been uploaded in the e-commerce blog by Vinayak Tadas who was a developer in the team that developed the product. the original post is here. I have attached the same code samples in this entry and can be downloaded for reference (without any implicit or explicit warranty or support from either me or Microsoft Corp. whatsoever : By downloading you accept all the risks involved without any liability on the part of Microsoft Corp. or myself.)

The attached CatalogWebCast_CodeSamples.zip file contains samples for:

Runtime Commerce Server APIs (those that are used to retrieve info for use in a e-commerce website)

• ProductCatalog: Getting products from a catalog along with its child products
• CatalogContext.GetCatalogs: Getting a set of catalogs from the current context
• CatalogContext.GetCategory: Getting the details of a category including child categories
• CatalogContext.GetProduct: Getting the details of a product and product family including the variants
• CatalogSearch: Searching the catalog with the basic properties specified

Design-time Commerce Server APIs (those that are used to create products, catalogs etc)

• CreateProperty: Creating a CatalogProperty using the CreateProperty method of the CatalogContext
• CatalogEnumerationProperty: Creating a CatalogEnumerationProperty using the overloaded CreateProperty method
• CreateDefinition:
  • Creating a CatalogDefinition using the CreateDefinition of CatalogContext
  • Creating a ProductDefinition using the CreateDefinition of CatalogContext
• .... and lots more...

Get the attachment:

Though we can have a long philosophical discussion as to which design to use, my intention here is to demonstrate and give some code sample as to how to achieve this. Some of the assumption for this are:

• All the records which contain the info to be submitted to an external web service is in a common buffer
• Each thread will process multiple records at a time before getting additional data from the buffer for the next iteration
• The code is not optimized for performance and can be worked upon

Ok, now with the disclaimers out of the way, lets see the basic steps involved in cobbling up a multi thread routine to call a web service. Broadly, we need to perform the following steps:

1. GetData(): Get the data from the buffer for each of the threads that will call the web service
2. Process(): Call the web service using the web proxy
3. SetData(): Merge the output obtained by each of the threads to a common output buffer

The three main things that need to be kept in mind are:

1. Using MUTEX to make sure that no two threads get the same set of data for the input.
2. Using MUTEX to make sure that no thread overwrites the output of the other.
3. Making sure that no thread gets zombied and that the main calling thread waits for the others to terminate.

The complete code

    /// <summary>
    /// Multi Threaded web processor
    /// </summary>
    public sealed class WebProcessingService
    {
        // setup the mutex for the input buffer
        private readonly static Mutex getDataSyncMutex = new Mutex(false);

        // setup the mutex for the output
        private readonly static Mutex setDataSyncMutex = new Mutex(false);

        // this indexer is used to make sure that the input array bounds are not violated
        // when the threads are getting fed the input
        private static int indexer = 0;

        string[] inputBuffer = null;
        //Assuming that the records to be processed result in a unique output
        Dictionary<string, string> outputBuffer = null;

        // else we can use the below, but we need to have mechanism to have a unique key for
        // each of the input records. If uniqueness is not necessary, we can just use a List<T>
        //Dictionary<string, SomeClass> outputBuffer = null;

        
        /// <summary>
        /// Initializes a new instance of the <see cref="WebProcessingService"/> class.
        /// </summary>
        public WebProcessingService(string[] inputRecords)
        {
            this.BuildInternalBuffers(inputRecords);
        }
        

        /// <summary>
        /// Builds the internal buffers, both the input and output buffer
        /// </summary>
        /// <param name="inputRecords">Input records</param>
        private void BuildInternalBuffers(string[] inputRecords)
        {
            this.inputBuffer = inputRecords;
            outputBuffer = new Dictionary<string, string>(inputRecords.Length);
            indexer = 0;            
            
        }

        /// <summary>
        /// Method initiates the mutli threaded processing of web calls
        /// </summary>
        /// <returns></returns>
        public Dictionary<string, string> ProcessRecords()
        {
            
            Thread[] pool = new Thread[Settings.Default.maxThreads];

            for (int i = 0; i < pool.Length; i++)
            {
                //spawn out a new thread
                Thread t = new Thread(new ThreadStart(ProcessCalls));
                //mark it as a non-background thread
                t.IsBackground = false;
                //start
                t.Start();
                //save the reference so that the main thread waits for its termination
                pool[i] = t;
            }
            //make the caller wait for all the threads to terminate
            foreach (Thread t in pool)
            {
                t.Join();
            }
            //return the output
            return outputBuffer;

        }

        /// <summary>
        /// Gets the data.
        /// </summary>
        /// <returns></returns>
        private string[] GetData()
        {
            // apply muetx to make sure that only one thread enter the region
            getDataSyncMutex.WaitOne();
            
            try
            {
                //init the temp buffers to copy from the input records
                string[] tempBuffer = new string[Settings.Default.recordsPerThreadPerIteration];
                //check if the indexer is withing the array bound and that i is within the bounds
                // of the temp buffer
                for (int i = 0; i < tempBuffer.Length && indexer < this.inputBuffer.Length; 
                    i++, indexer++)
                {
                    tempBuffer[i] = this.inputBuffer[indexer];
                }
                //now check if the indexer has gone out of bounds of the length of the buffer
                // this means that all input records have been passed onto the threads
                if (indexer > this.inputBuffer.Length)
                {
                    //signal end of input
                    return null;
                }
                    //if the indexer has not yet reached the end, check if it is at the boundary
                    // if so increment it so that the next thread entering gets the signal of 
                    // end-of-input
                else if (indexer == this.inputBuffer.Length)
                {                    
                    
                    indexer++;
                }
                // the remaining case is when the indexer is < this.inputBuffer.Length
                // which is safe as it means that the next thread will get input records
                return tempBuffer;
                
            }
            finally
            {
                //even if there is some exception, this will release the lock
                getDataSyncMutex.ReleaseMutex();
            }
        }

        /// <summary>
        /// Processes the web calls.
        /// </summary>
        private void ProcessCalls()
        {
            //instantiate the proxy
            FooWebService webProxy = new FooWebService();
            //instantiate the temp buffer
            string[] buffer = null;
            while (true)
            {
                //clear it out for the start of each iteration
                buffer = null;
                //get the data
                buffer = this.GetData();
                //if end-of-input is specified, done!
                if (buffer == null)
                {
                    break;
                }
                string[] outTempBuffer = new string[buffer.Length];
                // process each of the elem
                for (int i = 0; i < buffer.Length; i++)
                {
                    if (buffer[i] != null)
                    {
                        outTempBuffer[i] =
                            webProxy.SomeMetho(buffer[i]);                        
                    }
                }
                //once we get the current set out, feed into the output dictionary
                this.SetData(outTempBuffer);
            }
            
        }

        /// <summary>
        /// Sets the data.
        /// </summary>
        /// <param name="tempBuffer">The output record.</param>
        private void SetData(string[] tempBuffer)
        {
            //apply lock for mutual exlusion
            setDataSyncMutex.WaitOne();
            try
            {
                for (int i = 0; i < tempBuffer.Length; i++)
                {
                    //check if the key isnt already present before trying to add
                    if (tempBuffer[i] != null
                        && !string.IsNullOrEmpty(tempBuffer[i])
                        && !outputBuffer.ContainsKey(tempBuffer[i]))
                    {
                        //add if not present
                        outputBuffer.Add(tempBuffer[i],tempBuffer[i]);
                    }
                }

            }
            finally
            {
                //release lock
                setDataSyncMutex.ReleaseMutex();
            }
        }
    }