Azure blob storage as primary storage - Content-Length HTTP header is missing

TLDR: Trying to use azure as primary storage. Without encryption, can't upload to azure at all. With encryption, uploads work but it duplicates all files with one encrypted and one unencrypted copy.

Steps to reproduce

1. Configure azure blob storage as shown in https://github.com/nextcloud/server/pull/9483
2. Test uploading files with and without the default encryption module enabled

Expected behaviour

Azure should be used, and files uploaded to blob storage

Actual behaviour

Without encryption module enabled:

Files fail to upload to azure, presumably because they are missing the content length header

{"reqId":"kdSDvQDiCcsUWMPi3FWM","level":3,"time":"2019-02-26T16:23:00+00:00","remoteAddr":"censored","user":"grant","app":"no app in context","method":"PUT","url":"\/remote.php\/dav\/files\/grant\/censored","message":{"Exception":"MicrosoftAzure\\Storage\\Common\\Exceptions\\ServiceException","Message":"Fail:\nCode: 411\nValue: Content-Length HTTP header is missing.\ndetails (if any): \ufeff<?xml version=\"1.0\" encoding=\"utf-8\"?><Error><Code>MissingContentLengthHeader<\/Code><Message>Content-Length HTTP header is missing.\nRequestId:00dfc2bc-901e-00ab-71ef-cd6d13000000\nTime:2019-02-26T16:22:59.9955829Z<\/Message><HeaderName>Content-Length<\/HeaderName><\/Error>.","Code":411,"Trace":[{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/microsoft\/azure-storage-common\/src\/Common\/Internal\/ServiceRestProxy.php","line":407,"function":"throwIfError","class":"MicrosoftAzure\\Storage\\Common\\Internal\\ServiceRestProxy","type":"::","args":[{"__class__":"GuzzleHttp\\Psr7\\Response"},201]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/promises\/src\/Promise.php","line":203,"function":"MicrosoftAzure\\Storage\\Common\\Internal\\{closure}","class":"MicrosoftAzure\\Storage\\Common\\Internal\\ServiceRestProxy","type":"->","args":["*** sensitive parameters replaced ***"]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/promises\/src\/Promise.php","line":174,"function":"callHandler","class":"GuzzleHttp\\Promise\\Promise","type":"::","args":[2,"*** sensitive parameter replaced ***",[{"__class__":"GuzzleHttp\\Promise\\Promise"},{"__class__":"Closure"},{"__class__":"Closure"}]]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/promises\/src\/RejectedPromise.php","line":40,"function":"GuzzleHttp\\Promise\\{closure}","class":"GuzzleHttp\\Promise\\Promise","type":"::","args":["*** sensitive parameters replaced ***"]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/promises\/src\/TaskQueue.php","line":47,"function":"GuzzleHttp\\Promise\\{closure}","class":"GuzzleHttp\\Promise\\RejectedPromise","type":"::","args":["*** sensitive parameters replaced ***"]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/guzzle\/src\/Handler\/CurlMultiHandler.php","line":98,"function":"run","class":"GuzzleHttp\\Promise\\TaskQueue","type":"->","args":[]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/guzzle\/src\/Handler\/CurlMultiHandler.php","line":125,"function":"tick","class":"GuzzleHttp\\Handler\\CurlMultiHandler","type":"->","args":[]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/promises\/src\/Promise.php","line":246,"function":"execute","class":"GuzzleHttp\\Handler\\CurlMultiHandler","type":"->","args":[true]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/promises\/src\/Promise.php","line":223,"function":"invokeWaitFn","class":"GuzzleHttp\\Promise\\Promise","type":"->","args":[]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/promises\/src\/Promise.php","line":267,"function":"waitIfPending","class":"GuzzleHttp\\Promise\\Promise","type":"->","args":[]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/promises\/src\/Promise.php","line":225,"function":"invokeWaitList","class":"GuzzleHttp\\Promise\\Promise","type":"->","args":[]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/guzzlehttp\/promises\/src\/Promise.php","line":62,"function":"waitIfPending","class":"GuzzleHttp\\Promise\\Promise","type":"->","args":[]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/microsoft\/azure-storage-blob\/src\/Blob\/BlobRestProxy.php","line":1755,"function":"wait","class":"GuzzleHttp\\Promise\\Promise","type":"->","args":[]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/lib\/private\/Files\/ObjectStore\/Azure.php","line":107,"function":"createBlockBlob","class":"MicrosoftAzure\\Storage\\Blob\\BlobRestProxy","type":"->","args":["nextclouddata","urn:oid:636",null]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/lib\/private\/Files\/ObjectStore\/ObjectStoreStorage.php","line":452,"function":"writeObject","class":"OC\\Files\\ObjectStore\\Azure","type":"->","args":["urn:oid:636",null]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/lib\/private\/Files\/Storage\/Wrapper\/Wrapper.php","line":630,"function":"writeStream","class":"OC\\Files\\ObjectStore\\ObjectStoreStorage","type":"->","args":["files\/censored",null,null]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/lib\/private\/Files\/Storage\/Wrapper\/Wrapper.php","line":630,"function":"writeStream","class":"OC\\Files\\Storage\\Wrapper\\Wrapper","type":"->","args":["files\/censored",null,null]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/lib\/private\/Files\/Storage\/Wrapper\/Wrapper.php","line":630,"function":"writeStream","class":"OC\\Files\\Storage\\Wrapper\\Wrapper","type":"->","args":["files\/censored",null,null]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/lib\/private\/Files\/Storage\/Wrapper\/Wrapper.php","line":630,"function":"writeStream","class":"OC\\Files\\Storage\\Wrapper\\Wrapper","type":"->","args":["files\/censored",null,null]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/apps\/dav\/lib\/Connector\/Sabre\/File.php","line":169,"function":"writeStream","class":"OC\\Files\\Storage\\Wrapper\\Wrapper","type":"->","args":["files\/censored",null]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/apps\/dav\/lib\/Connector\/Sabre\/Directory.php","line":156,"function":"put","class":"OCA\\DAV\\Connector\\Sabre\\File","type":"->","args":[null]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/sabre\/dav\/lib\/DAV\/Server.php","line":1096,"function":"createFile","class":"OCA\\DAV\\Connector\\Sabre\\Directory","type":"->","args":["censoredfilename",null]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/sabre\/dav\/lib\/DAV\/CorePlugin.php","line":525,"function":"createFile","class":"Sabre\\DAV\\Server","type":"->","args":["files\/grant\/censored",null,null]},{"function":"httpPut","class":"Sabre\\DAV\\CorePlugin","type":"->","args":[{"absoluteUrl":"https:\/\/my.server.com\/remote.php\/dav\/files\/grant\/censored","__class__":"Sabre\\HTTP\\Request"},{"__class__":"Sabre\\HTTP\\Response"}]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/sabre\/event\/lib\/EventEmitterTrait.php","line":105,"function":"call_user_func_array","args":[[{"__class__":"Sabre\\DAV\\CorePlugin"},"httpPut"],[{"absoluteUrl":"https:\/\/my.server.com\/remote.php\/dav\/files\/grant\/censored","__class__":"Sabre\\HTTP\\Request"},{"__class__":"Sabre\\HTTP\\Response"}]]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/sabre\/dav\/lib\/DAV\/Server.php","line":479,"function":"emit","class":"Sabre\\Event\\EventEmitter","type":"->","args":["method:PUT",[{"absoluteUrl":"https:\/\/my.server.com\/remote.php\/dav\/files\/grant\/censored","__class__":"Sabre\\HTTP\\Request"},{"__class__":"Sabre\\HTTP\\Response"}]]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/sabre\/dav\/lib\/DAV\/Server.php","line":254,"function":"invokeMethod","class":"Sabre\\DAV\\Server","type":"->","args":[{"absoluteUrl":"https:\/\/my.server.com\/remote.php\/dav\/files\/grant\/censored","__class__":"Sabre\\HTTP\\Request"},{"__class__":"Sabre\\HTTP\\Response"}]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/apps\/dav\/lib\/Server.php","line":301,"function":"exec","class":"Sabre\\DAV\\Server","type":"->","args":[]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/apps\/dav\/appinfo\/v2\/remote.php","line":35,"function":"exec","class":"OCA\\DAV\\Server","type":"->","args":[]},{"file":"\/snap\/nextcloud\/11688\/htdocs\/remote.php","line":163,"args":["\/snap\/nextcloud\/11688\/htdocs\/apps\/dav\/appinfo\/v2\/remote.php"],"function":"require_once"}],"File":"\/snap\/nextcloud\/11688\/htdocs\/3rdparty\/microsoft\/azure-storage-common\/src\/Common\/Internal\/ServiceRestProxy.php","Line":490,"CustomMessage":"--"},"userAgent":"Mozilla\/5.0 (Windows) mirall\/2.5.1final (build 20181204) (Nextcloud)","version":"15.0.4.0"}

With encryption module enabled:

Files upload just fine! Awesome!

Except that it's actually storing each file twice in the container - once encrypted, and once unencrypted. The unencrypted copy stays there even after completely removing the file from nextcloud. The encrypted copy does get properly deleted.

I suspect that enabling the encryption module changes how the file is handled, instead of being streamed to azure, it's encrypted and a new transfer to azure happens, and that one has the proper headers?

It's so close to working properly that it's driving me crazy!

Server configuration

Operating system: Ubuntu 18.04

Web server/Database/PHP version: Everything configured by snap

Nextcloud version: 15.0.4.0

Updated from an older Nextcloud/ownCloud or fresh install: Fresh

Where did you install Nextcloud from: snap

Signing status:

No errors have been found.

List of activated apps:

Enabled:
  - accessibility: 1.1.0
  - activity: 2.8.2
  - cloud_federation_api: 0.1.0
  - dav: 1.8.1
  - encryption: 2.3.0
  - federatedfilesharing: 1.5.0
  - federation: 1.5.0
  - files: 1.10.0
  - files_external: 1.6.0
  - files_pdfviewer: 1.4.0
  - files_rightclick: 0.11.0
  - files_sharing: 1.7.0
  - files_texteditor: 2.7.0
  - files_trashbin: 1.5.0
  - files_versions: 1.8.0
  - files_videoplayer: 1.4.0
  - firstrunwizard: 2.4.0
  - gallery: 18.2.0
  - keeporsweep: 0.2.1
  - logreader: 2.0.0
  - lookup_server_connector: 1.3.0
  - nextcloud_announcements: 1.4.0
  - notes: 2.5.1
  - notifications: 2.3.0
  - oauth2: 1.3.0
  - password_policy: 1.5.0
  - provisioning_api: 1.5.0
  - quota_warning: 1.4.0
  - serverinfo: 1.5.0
  - sharebymail: 1.5.0
  - support: 1.0.0
  - survey_client: 1.3.0
  - systemtags: 1.5.0
  - twofactor_backupcodes: 1.4.1
  - twofactor_u2f: 2.1.1
  - workflowengine: 1.5.0
Disabled:
  - admin_audit
  - comments
  - cospend
  - files_downloadactivity
  - files_external_gdrive
  - files_markdown
  - files_mindmap
  - metadata
  - news
  - polls
  - quicknotes
  - ransomware_protection
  - tasks
  - terms_of_service
  - theming
  - twofactor_totp
  - user_ldap

Nextcloud configuration:

{
    "system": {
        "objectstore": {
            "class": "OC\\Files\\ObjectStore\\Azure",
            "arguments": {
                "account_name": "myazurestorageaccount",
                "account_key": "myazurestoragekey",
                "container": "nameofcontainer"
            }
        },
        "apps_paths": [
            {
                "path": "\/snap\/nextcloud\/current\/htdocs\/apps",
                "url": "\/apps",
                "writable": false
            },
            {
                "path": "\/var\/snap\/nextcloud\/current\/nextcloud\/extra-apps",
                "url": "\/extra-apps",
                "writable": true
            }
        ],
        "supportedDatabases": [
            "mysql"
        ],
        "memcache.locking": "\\OC\\Memcache\\Redis",
        "memcache.local": "\\OC\\Memcache\\Redis",
        "redis": {
            "host": "***REMOVED SENSITIVE VALUE***",
            "port": 0
        },
        "instanceid": "***REMOVED SENSITIVE VALUE***",
        "passwordsalt": "***REMOVED SENSITIVE VALUE***",
        "secret": "***REMOVED SENSITIVE VALUE***",
        "trusted_domains": [
            "cloud.emsley.ca"
        ],
        "datadirectory": "***REMOVED SENSITIVE VALUE***",
        "dbtype": "mysql",
        "version": "15.0.4.0",
        "overwrite.cli.url": "http:\/\/my.server.com",
        "dbname": "***REMOVED SENSITIVE VALUE***",
        "dbhost": "***REMOVED SENSITIVE VALUE***",
        "dbport": "",
        "dbtableprefix": "oc_",
        "mysql.utf8mb4": true,
        "dbuser": "***REMOVED SENSITIVE VALUE***",
        "dbpassword": "***REMOVED SENSITIVE VALUE***",
        "installed": true,
        "mail_smtpmode": "smtp",
        "mail_smtpsecure": "tls",
        "mail_sendmailmode": "smtp",
        "mail_from_address": "***REMOVED SENSITIVE VALUE***",
        "mail_domain": "***REMOVED SENSITIVE VALUE***",
        "mail_smtpauthtype": "PLAIN",
        "mail_smtpauth": 1,
        "mail_smtphost": "***REMOVED SENSITIVE VALUE***",
        "mail_smtpport": "587",
        "mail_smtpname": "***REMOVED SENSITIVE VALUE***",
        "mail_smtppassword": "***REMOVED SENSITIVE VALUE***",
        "maintenance": false,
        "twofactor_enforced": "false",
        "twofactor_enforced_groups": [],
        "twofactor_enforced_excluded_groups": []
    }
}

Are you using external storage, if yes which one: Azure

Are you using encryption: Trying to

Are you using an external user-backend, if yes which one: No

Client configuration

Browser: Tried Chrome and Firefox

Operating system: Windows 10

Logs

Web server error log

Nextcloud log (data/nextcloud.log)

Browser log

Start with lib/private/Files/ObjectStore/Azure.php, especially the writeObject call at line 107 shown in the trace, then compare the upload paths with encryption enabled and disabled. Reproduce the missing Content-Length failure and verify that successful uploads do not leave an extra unencrypted blob after deletion.