Cloud Filter API - CF_HYDRATION_POLICY_PROGRESSIVE

Question

While working with cfapi.h, I've tried to map random access over a cloud file. I'm registering the SyncRoot as:

    CF_SYNC_POLICIES CfSyncPolicies = { 0 };  
    CfSyncPolicies.StructSize = sizeof(CF_SYNC_POLICIES);  
    CfSyncPolicies.HardLink = CF_HARDLINK_POLICY_NONE;  
    CfSyncPolicies.Hydration.Primary = CF_HYDRATION_POLICY_PROGRESSIVE;  
    CfSyncPolicies.Hydration.Modifier = CF_HYDRATION_POLICY_MODIFIER_STREAMING_ALLOWED | CF_HYDRATION_POLICY_MODIFIER_AUTO_DEHYDRATION_ALLOWED;  

...

The problem is that OnFetchData requires the entire contents of the file, even if I read only a few bytes.

Is there a way to return less bytes than asked by CF_CALLBACK_PARAMETERS.FetchData.RequiredLength? For example, if RequiredLength is 100Mb, how can I return only 64Kb and force another call, if needed, to OnFetchData?

Bellow is my OnFetchData function:

void CALLBACK OnFetchData(CONST CF_CALLBACK_INFO *callbackInfo, CONST CF_CALLBACK_PARAMETERS *callbackParameters) {  
    struct fuse *f = (struct fuse *)callbackInfo->CallbackContext;  
    fuse_lock(f);  
    f->fetch_op_count ++;  
    fuse_unlock(f);  
  
    CF_OPERATION_INFO opInfo = { 0 };  
    CF_OPERATION_PARAMETERS opParams = { 0 };  
  
    opInfo.StructSize = sizeof(opInfo);  
    opInfo.Type = CF_OPERATION_TYPE_TRANSFER_DATA;  
    opInfo.ConnectionKey = callbackInfo->ConnectionKey;  
    opInfo.TransferKey = callbackInfo->TransferKey;  
    opInfo.RequestKey = callbackInfo->RequestKey;  
    opInfo.CorrelationVector = callbackInfo->CorrelationVector;  
  
    opParams.ParamSize = CF_SIZE_OF_OP_PARAM(TransferData);  
    opParams.TransferData.CompletionStatus = STATUS_SUCCESS;  
    opParams.TransferData.Offset.QuadPart = 0;  
    opParams.TransferData.Length.QuadPart = 0;  
  
    int open_err = 0;  
    struct fuse_file_info finfo = { 0 };  
    char *path = toUTF8_path((wchar_t *)callbackInfo->FileIdentity, callbackInfo->FileIdentityLength);  
  
    DEBUG_DUMP("FetchData %s", path);  
  
    if (f->op.open) {  
        fuse_lock(f);  
        open_err = f->op.open(path, &finfo);  
        fuse_unlock(f);  
    }  
  
    DEBUG_ERRNO("op.open", open_err);  
  
    char buf[0x10000];  
    HRESULT hr;  
    if (open_err) {  
        opParams.TransferData.Buffer = buf;  
        opParams.TransferData.Offset.QuadPart = callbackParameters->FetchData.RequiredFileOffset.QuadPart;  
        opParams.TransferData.Length.QuadPart = callbackParameters->FetchData.RequiredLength.QuadPart;  
        opParams.TransferData.CompletionStatus = STATUS_UNSUCCESSFUL;  
  
        hr = CfExecute(&opInfo, &opParams);  
        DEBUG_HANDLE("CfExecute", hr);  
    } else {  
        if (f->op.read) {  
            __int64 size = (__int64)callbackParameters->FetchData.RequiredLength.QuadPart;  
            __int64 offset = (__int64)callbackParameters->FetchData.RequiredFileOffset.QuadPart;  
  
            LARGE_INTEGER ProviderProgressCompleted = { 0 };  
  
            __int64 read_size = size;  
            if (read_size > sizeof(buf))  
                read_size = sizeof(buf);  
  
            do {  
                int err = f->op.read(path, buf, read_size, offset, &finfo);  
                DEBUG_ERRNO2("op.read", err);  
                if (err < 0) {  
                    DEBUG_ERRNO("op.read", err);  
                    opParams.TransferData.CompletionStatus = NTSTATUS_FROM_WIN32(EIO);  
                    opParams.TransferData.Buffer = buf;  
                    opParams.TransferData.Offset.QuadPart = callbackParameters->FetchData.RequiredFileOffset.QuadPart;//offset;  
                    opParams.TransferData.Length.QuadPart = callbackParameters->FetchData.RequiredLength.QuadPart;//size;  
                    hr = CfExecute(&opInfo, &opParams);  
  
                    DEBUG_HANDLE("CfExecute", hr);  
                    break;  
                } else {  
                    opParams.TransferData.Buffer = buf;  
                    opParams.TransferData.Offset.QuadPart = offset;  
                    opParams.TransferData.Length.QuadPart = err;  
                    opParams.TransferData.CompletionStatus = STATUS_SUCCESS;  
                      
                    hr = CfExecute(&opInfo, &opParams);  
                    DEBUG_HANDLE("CfExecute", hr);  
                    if (FAILED(hr)) {  
                        // 0x8007018e: The cloud operation was canceled by user.  
                        if (hr != 0x8007018e) {  
                            DEBUG_DUMP("Error in CfExecute => offset %" PRId64 ", len %i", offset, err);  
                            opParams.TransferData.CompletionStatus = NTSTATUS_FROM_WIN32(EIO);  
                            opParams.TransferData.Buffer = buf;  
                            opParams.TransferData.Offset.QuadPart = callbackParameters->FetchData.RequiredFileOffset.QuadPart;  
                            opParams.TransferData.Length.QuadPart = callbackParameters->FetchData.RequiredLength.QuadPart;  
                            hr = CfExecute(&opInfo, &opParams);  
                        }  
                        break;  
                    }  
  
                    ProviderProgressCompleted.QuadPart += err;  
                    hr = CfReportProviderProgress(callbackInfo->ConnectionKey, callbackInfo->TransferKey, callbackParameters->FetchData.RequiredLength, ProviderProgressCompleted);  
                    DEBUG_HANDLE("CfReportProviderProgress", hr);  
                    DEBUG_DUMP("  [ %3.2f%% ]", (double)ProviderProgressCompleted.QuadPart / (double)callbackParameters->FetchData.RequiredLength.QuadPart * 100);  
  
                    size -= err;  
                    offset += err;  
  
                    if (size < read_size)  
                        read_size = size;  
                }  
            } while (size > 0);  
        }  
  
        if (f->op.release) {  
            fuse_lock(f);  
            f->op.release(path, &finfo);  
            fuse_unlock(f);  
        }  
    }  
    free(path);  
    fuse_lock(f);  
    f->fetch_op_count --;  
    fuse_unlock(f);  
}  

Answer 1

For me the reason of full hydration was Microsoft Defender Real-time protection, if I turn it off the feature starts working as described

Answer 2

Hi @Eduard Suica

I suggest you could refer to the official sample Cloud Mirror, there is CHUNKSIZE to control the size of each chunk to be transferred and chunk buffer to be allocated.

// Allocate the buffer used in the overlapped read.  
    chunkBufferSize = (ULONG)min(requiredLength.QuadPart, CHUNKSIZE);