On Friday, 9 October 2015 at 04:04:57 UTC, holo wrote:

> 	r.dateString = 	client.addRequestHeader("Authoryzation:", "AWS4-HMAC-SHA256" ~ " " ~ "Credential=" ~ accKey ~ "/" ~ xamztime ~ "/" ~ zone ~ "/" ~ service ~ "/" ~ "aws4_request" ~ ", " ~ "SignedHeaders=" ~ "content-type;host;x-amz-date" ~ ", " ~ "Signature=" ~ signature);

authorisation ??? (Or maybe authorization) ie a typo (check other fields carefully too, just in case)



> 	auto url = service ~ ".amazonaws.com?" ~ "Action=DescribeInstances&Version=2013-10-15";
> 	writeln(url);
> 	auto content = get(url, client);
> 	writeln(content);
> }
>
> Everything is compiling but im getting 400 (Bad Request):
>
>   [root@ultraxps aws]# ./header.d
>
> action=DescribeRegions&version=2013-10-15
>
> content-type: application/x-www-form-urlencoded; charset=utf-8
> host: ec2.amazonaws.com
> x-amz-date: 20151009T053800Z
>
>
> AWS4-HMAC-SHA256
> 20151009T053800Z
> 20151009/us-east-1/ec2/aws4_request
> 888595748692147ceafafcae3941ec0d83ac42c97641e4d954d7447a00c56270
>
> 69b1e4c5212cc6b485569fdfb43f7dde94413b36c50393c55d4810ced47f167b
>
> ec2.amazonaws.com?Action=DescribeRegions&Version=2013-10-15
> std.net.curl.CurlException@/usr/include/dlang/dmd/std/net/curl.d(824): HTTP request returned status code 400 (Bad Request)
> ----------------
> /tmp/.rdmd-0/rdmd-header.d-54C0A2BD6BD71C27D9AC7319D786A6F3/header(pure @safe bool std.exception.enforce!(std.net.curl.CurlException, bool).enforce(bool, lazy const(char)[], immutable(char)[], ulong)+0x65) [0x5004bd]
> /tmp/.rdmd-0/rdmd-header.d-54C0A2BD6BD71C27D9AC7319D786A6F3/header(char[] std.net.curl._basicHTTP!(char)._basicHTTP(const(char)[], const(void)[], std.net.curl.HTTP)+0x1a2) [0x4fc0ba]
> /tmp/.rdmd-0/rdmd-header.d-54C0A2BD6BD71C27D9AC7319D786A6F3/header(char[] std.net.curl.get!(std.net.curl.HTTP, char).get(const(char)[], std.net.curl.HTTP)+0x6a) [0x4fbec2]
> /tmp/.rdmd-0/rdmd-header.d-54C0A2BD6BD71C27D9AC7319D786A6F3/header(_Dmain+0x840) [0x4f8f98]
> /tmp/.rdmd-0/rdmd-header.d-54C0A2BD6BD71C27D9AC7319D786A6F3/header(_D2rt6dmain211_d_run_mainUiPPaPUAAaZiZ6runAllMFZ9__lambda1MFZv+0x1f) [0x533eab]
> /tmp/.rdmd-0/rdmd-header.d-54C0A2BD6BD71C27D9AC7319D786A6F3/header(void rt.dmain2._d_run_main(int, char**, extern (C) int function(char[][])*).tryExec(scope void delegate())+0x2a) [0x533e06]
> /tmp/.rdmd-0/rdmd-header.d-54C0A2BD6BD71C27D9AC7319D786A6F3/header(void rt.dmain2._d_run_main(int, char**, extern (C) int function(char[][])*).runAll()+0x2b) [0x533e67]
> /tmp/.rdmd-0/rdmd-header.d-54C0A2BD6BD71C27D9AC7319D786A6F3/header(void rt.dmain2._d_run_main(int, char**, extern (C) int function(char[][])*).tryExec(scope void delegate())+0x2a) [0x533e06]
> /tmp/.rdmd-0/rdmd-header.d-54C0A2BD6BD71C27D9AC7319D786A6F3/header(_d_run_main+0x1d2) [0x533d86]
> /tmp/.rdmd-0/rdmd-header.d-54C0A2BD6BD71C27D9AC7319D786A6F3/header(main+0x12) [0x52cb62]
> /usr/lib/libc.so.6(__libc_start_main+0xf0) [0x7f9c39f71610]
> [root@ultraxps aws]#
>
> What am i doing wrong? What is strange when when i do same with curl its responding normally of course is not authenticated).
>
> [root@ultraxps aws]# curl ec2.amazonaws.com?Action=DescribeInstances&Version=2013-10-15
> [1] 20390
> [root@ultraxps aws]# <?xml version="1.0" encoding="UTF-8"?>
> <Response><Errors><Error><Code>MissingParameter</Code><Message>The request must contain the parameter AWSAccessKeyId</Message></Error></Errors><RequestID>e1352781-c2b4-4e74-ade3-80d655efd0ac</RequestID></Response>

I correct typo (thats whats happening when you trying to solve problem after night at 6:00 AM ).But still  have same error. When use curl with same generated creditentials im getting "400". What is strange when im using same generated headers and signature with command line curl im getting "500". My headers right now:

auto client = HTTP();
        client.clearRequestHeaders;
        client.addRequestHeader("Content-Type:", "application/x-www-form-urlencoded; charset=utf-8");
        client.addRequestHeader("Host:", service ~ ".amazonaws.com");
        client.addRequestHeader("X-Amz-Date:", xamztime);
        client.addRequestHeader("Authorization:", "AWS4-HMAC-SHA256" ~ " " ~ "Credential=" ~ accKey ~ "/" ~ xamztime ~ "/" ~ zone ~ "/" ~ service ~ "/" ~ "aws4_request" ~ ", " ~ "SignedHeaders=" ~ "content-type;host;x-amz-date" ~ ", " ~ "Signature=" ~ signature);

        auto url = service ~ ".amazonaws.com?" ~ "Action=DescribeInstances&Version=2013-10-15";
        writeln(url);
        auto content = get(url, client);
        writeln(content);


Did i set up it correctly?

I doped HTTP packets with wireshark, maybe that  will  be useful:

First recived:

 Hypertext Transfer Protocol
    HTTP/1.1 400 Bad Request\r\n
        [Expert Info (Chat/Sequence): HTTP/1.1 400 Bad Request\r\n]
        Request Version: HTTP/1.1
        Status Code: 400
        Response Phrase: Bad Request
    Transfer-Encoding: chunked\r\n
    Date: Fri, 09 Oct 2015 15:43:37 GMT\r\n
    Cneonction: close\r\n
    Server: AmazonEC2\r\n
    \r\n
    [HTTP response 1/2]
    [Next request in frame: 7739]
    HTTP chunked response

Second sent:

Hypertext Transfer Protocol
    GET /?Action=DescribeInstances&Version=2013-10-15 HTTP/1.1\r\n
        [Expert Info (Chat/Sequence): GET /?Action=DescribeInstances&Version=2013-10-15 HTTP/1.1\r\n]
        Request Method: GET
        Request URI: /?Action=DescribeInstances&Version=2013-10-15
        Request Version: HTTP/1.1
    Host:: ec2.amazonaws.com\r\n
    User-Agent: Phobos-std.net.curl/2.069 (libcurl/7.44.0)\r\n
    Accept: */*\r\n
    Content-Type:: application/x-www-form-urlencoded; charset=utf-8\r\n
    X-Amz-Date:: 20151009T174337Z\r\n
    Authorization:: AWS4-HMAC-SHA256 Credential=AKIAIHGPAJQ2PJ47YKKA/20151009T174337Z/us-east-1/ec2/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=3049eff7d6b5084c2fee71d10d5cffb17e0df1a9fe76963859c423f637d0496e\r\n
    \r\n
    [Full request URI: http://: ec2.amazonaws.com/?Action=DescribeInstances&Version=2013-10-15]
    [HTTP request 2/2]

What is strange why im getting first that 400 Bad Request packet and after that is sent GET one?

On Friday, 9 October 2015 at 16:01:33 UTC, holo wrote:
>     Host:: ec2.amazonaws.com\r\n

This, ...

>     Content-Type:: application/x-www-form-urlencoded; charset=utf-8\r\n
>     X-Amz-Date:: 20151009T174337Z\r\n
>     Authorization:: AWS4-HMAC-SHA256 Credential=AKIAIHGPAJQ2PJ47YKKA/20151009T174337Z/us-east-1/ec2/aws4_request, SignedHeaders=content-type;host;x-amz-date, Signature=3049eff7d6b5084c2fee71d10d5cffb17e0df1a9fe76963859c423f637d0496e\r\n
>     \r\n

... these ones, ...

>     [Full request URI: http://: ec2.amazonaws.com/?Action=DescribeInstances&Version=2013-10-15]
>     [HTTP request 2/2]

... and this look wrong. Where does the additional ": " come from?

OK i find out error, in addRequestHeader i was using ":" after header name what casing problem. I removed it and now im getting "unauthorized". Here is how it looks right now:

 HTTP/1.1 401 Unauthorized\r\n
        [Expert Info (Chat/Sequence): HTTP/1.1 401 Unauthorized\r\n]
        Request Version: HTTP/1.1
        Status Code: 401
        Response Phrase: Unauthorized
    Transfer-Encoding: chunked\r\n
    Date: Fri, 09 Oct 2015 16:22:47 GMT\r\n
    Server: AmazonEC2\r\n
    \r\n
    [HTTP response 1/2]
    [Next request in frame: 8371]
    HTTP chunked response
        Data chunk (254 octets)
            Chunk size: 254 octets
            Data (254 bytes)

In data field i can read:

"AWS was not able to validate provided access credentials" Is my signing process incorrect?

On Friday, 9 October 2015 at 16:30:26 UTC, holo wrote:
> OK i find out error, in addRequestHeader i was using ":" after header name what casing problem. I removed it and now im getting "unauthorized". Here is how it looks right now:
>
>  HTTP/1.1 401 Unauthorized\r\n
>         [Expert Info (Chat/Sequence): HTTP/1.1 401 Unauthorized\r\n]
>         Request Version: HTTP/1.1
>         Status Code: 401
>         Response Phrase: Unauthorized
>     Transfer-Encoding: chunked\r\n
>     Date: Fri, 09 Oct 2015 16:22:47 GMT\r\n
>     Server: AmazonEC2\r\n
>     \r\n
>     [HTTP response 1/2]
>     [Next request in frame: 8371]
>     HTTP chunked response
>         Data chunk (254 octets)
>             Chunk size: 254 octets
>             Data (254 bytes)
>
> In data field i can read:
>
> "AWS was not able to validate provided access credentials" Is my signing process incorrect?

Maybe i will put my present code again:

#!/usr/bin/rdmd -L-lcurl

import std.stdio;
import std.string;
import std.file;
import std.datetime;
import std.process;
import std.digest.sha;
import std.net.curl;
import std.uri;
import sigv4;


auto zone = "us-east-1";
auto service = "ec2";


void main()
{
	auto accKey = environment["AWS_ACCESS_KEY"];
	auto secKey = environment["AWS_SECRET_KEY"];

	auto currentClock = Clock.currTime;

	auto currentDate = cast(Date)currentClock;
	auto curDateStr = currentDate.toISOString;

	auto currentTime = cast(TimeOfDay)currentClock;
	auto curTimeStr = currentTime.toISOString;

	auto xamztime = curDateStr ~ "T" ~ curTimeStr ~ "Z";

	string[string] empty;

	SignableRequest r;
	r.dateString = curDateStr;
	r.timeStringUTC = curTimeStr;
	r.region = zone;
	r.service = service;
	r.canonicalRequest = CanonicalRequest(
			"POST",
			"/",
			["action" : "DescribeInstances", "version" : "2013-10-15"],
//			["accept" : "*/*",
			["content-type" : "application/x-www-form-urlencoded; charset=utf-8",
			 "host" : service ~ ".amazonaws.com",
			 "x-amz-date" : xamztime],
			 cast(ubyte[])"");          //cast(ubyte[])"Action=DescribeInstances&Version=2013-10-15");
	
	auto qParm = canonicalQueryString(r.canonicalRequest.queryParameters);

	auto sigHead = canonicalHeaders(r.canonicalRequest.headers);

	auto sigStr = signableString(r);

	auto sigKey = signingKey(secKey, curDateStr, zone, service);
	
	auto signature = sign(sigKey, cast(ubyte[])sigStr).toHexString().toLower();

	writeln();	
	writeln(qParm);
	writeln();
	writeln(sigHead);
	writeln();
	writeln(sigStr);
	writeln();
	writeln(signature);
	writeln();
	auto client = HTTP();
//	client.clearRequestHeaders;
	client.addRequestHeader("content-type", "application/x-www-form-urlencoded; charset=utf-8");
	client.addRequestHeader("host", service ~ ".amazonaws.com");
	client.addRequestHeader("x-amz-date", xamztime);
	client.addRequestHeader("authorization", "AWS4-HMAC-SHA256" ~ " " ~ "Credential=" ~ accKey ~ "/" ~ xamztime ~ "/" ~ zone ~ "/" ~ service ~ "/" ~ "aws4_request" ~ ", " ~ "SignedHeaders=" ~ "content-type;host;x-amz-date" ~ ", " ~ "Signature=" ~ signature);

	auto url = "ec2.amazonaws.com/?" ~ "Action=DescribeInstances&Version=2013-10-15";
	auto urlenc = encode(url);
	writeln(url);
	auto content = get(urlenc, client);
	writeln(content);
}

Is my signing process correct?

After long fight with previous code i try to rewrite "one-to-one" python example from http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html (GET part) from begging to D with full success. Here is working code in clear D. Im using hmac function which is available from 2.069 of phobos). It is not too beautiful but like i said im beginner, here it is:


#!/usr/bin/rdmd -L-lcurl

module sigawsv4;

import std.stdio, std.process;
import std.digest.sha, std.digest.hmac;
import std.string;
import std.conv;
import std.datetime;
import std.net.curl;

void main()
{
	auto accessKey = environment["AWS_ACCESS_KEY"];
	auto secretKey = environment["AWS_SECRET_KEY"];

	auto currentClock = Clock.currTime(UTC());

	auto currentDate = cast(Date)currentClock;
	auto curDateStr = currentDate.toISOString;

	auto currentTime = cast(TimeOfDay)currentClock;
	auto curTimeStr = currentTime.toISOString;

	auto xamztime = curDateStr ~ "T" ~ curTimeStr ~ "Z";

	
	string method = "GET";
	string service = "ec2";
	string host = "ec2.amazonaws.com";
	string region = "us-east-1";
	string endpoint = "https://ec2.amazonaws.com";
	string request_parameters = "Action=DescribeInstances&Version=2013-10-15";


	auto hmac_sha256(ubyte[] key, ubyte[] msg)
	{
		auto hmac = hmac!SHA256(key);
		hmac.put(msg);
		auto digest = hmac.finish;

		return digest;
	}

	alias sign = hmac_sha256;

	auto getSignatureKey(string key, string dateStamp, string regionName, string serviceName)
	{
		ubyte[] kString = cast(ubyte[])("AWS4" ~ key);
		auto kDate = sign(kString, cast(ubyte[])dateStamp);
		auto kRegion = sign(kDate, cast(ubyte[])regionName);
		auto kService = sign(kRegion,  cast(ubyte[])serviceName);
		auto kSigning = sign(kService, cast(ubyte[])"aws4_request");

		return kSigning;
	}


	string canonicalURI = "/";
	string canonicalQueryString = request_parameters;
	string canonicalHeadersString =  "host:" ~ host ~ "\n" ~ "x-amz-date:" ~ xamztime ~ "\n";
	string signedHeaders = "host;x-amz-date";
	string payloadHash = sha256Of("").toHexString.toLower;
	string canonicalRequest = method ~ "\n" ~ canonicalURI ~ "\n" ~ canonicalQueryString ~ "\n" ~ canonicalHeadersString ~ "\n" ~ signedHeaders ~ "\n" ~ payloadHash;


	string algorithm = "AWS4-HMAC-SHA256";
	string credentialScope = curDateStr ~ "/" ~ region ~ "/" ~ service ~ "/" ~ "aws4_request";
	string stringToSign = algorithm ~ "\n" ~ xamztime ~ "\n" ~ credentialScope ~ "\n" ~ sha256Of(canonicalRequest).toHexString.toLower;

	auto signingKey = getSignatureKey(secretKey, curDateStr, region, service);

	string signature = hmac_sha256(signingKey, cast(ubyte[])stringToSign).toHexString.toLower;

	string authorizationHeader = algorithm ~ " " ~ "Credential=" ~ accessKey ~ "/" ~ credentialScope ~ ", " ~ "SignedHeaders=" ~ signedHeaders ~ ", " ~ "Signature=" ~ signature;

	auto client = HTTP(endpoint ~ "?" ~ canonicalQueryString);
	client.method = HTTP.Method.get;
	client.addRequestHeader("x-amz-date", xamztime);
	client.addRequestHeader("Authorization", authorizationHeader);
	auto content = client.perform();

	writeln(content);
}

Now will try to "pack" it to some class and make it more usable and universal. I think that thread can be closed/solved (if something like this exist here)

Thank you all for help.

//holo

On Sunday, 11 October 2015 at 23:16:51 UTC, holo wrote:
> After long fight with previous code i try to rewrite "one-to-one" python example from http://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html (GET part) from begging to D with full success. Here is working code in clear D. Im using hmac function which is available from 2.069 of phobos). It is not too beautiful but like i said im beginner, here it is:

Thanks for sharing holo - you saved me a bit of time, as I needed just the same thing for another api (dnsmadeeasy).

Good luck with your own project.


Laeeth.

On Sunday, 11 October 2015 at 23:16:51 UTC, holo wrote:
> 	auto hmac_sha256(ubyte[] key, ubyte[] msg)
> 	{
> 		auto hmac = hmac!SHA256(key);
> 		hmac.put(msg);
> 		auto digest = hmac.finish;
>
> 		return digest;
> 	}
>
> 	alias sign = hmac_sha256;

std.digest.hmac already contains a helper function for this:

    hmac!SHA256(data, secret);

The order of arguments is this way to allow chaining with UFCS:

    data.hmac!SHA256(secret);

But it seems the documentation generator can't handle ddoc comments in templates, so unfortunately it doesn't appear in the documentation.

Thank you for info, i changed my code to use that build in template and changed "cast(ubyte[]"  to "xxx.representation" and it is still working:

#!/usr/bin/rdmd -L-lcurl

module sigawsv4;

import std.stdio, std.process;
import std.digest.sha, std.digest.hmac;
import std.string;
import std.conv;
import std.datetime;
import std.net.curl;

void main()
{
	auto accessKey = environment["AWS_ACCESS_KEY"];
	auto secretKey = environment["AWS_SECRET_KEY"];

	auto currentClock = Clock.currTime(UTC());

	auto currentDate = cast(Date)currentClock;
	auto curDateStr = currentDate.toISOString;

	auto currentTime = cast(TimeOfDay)currentClock;
	auto curTimeStr = currentTime.toISOString;

	auto xamztime = curDateStr ~ "T" ~ curTimeStr ~ "Z";

	
	string method = "GET";
	string service = "ec2";
	string host = "ec2.amazonaws.com";
	string region = "us-east-1";
	string endpoint = "https://ec2.amazonaws.com";
	string request_parameters = "Action=DescribeInstances&Version=2013-10-15";


	alias sign = hmac!SHA256;

	auto getSignatureKey(string key, string dateStamp, string regionName, string serviceName)
	{
		auto kString = ("AWS4" ~ key).representation;
		auto kDate = sign(dateStamp.representation, kString);
		auto kRegion = sign(regionName.representation, kDate);
		auto kService = sign(serviceName.representation, kRegion);
		auto kSigning = sign("aws4_request".representation, kService);

		return kSigning;
	}


	string canonicalURI = "/";
	string canonicalQueryString = request_parameters;
	string canonicalHeadersString =  "host:" ~ host ~ "\n" ~ "x-amz-date:" ~ xamztime ~ "\n";
	string signedHeaders = "host;x-amz-date";
	string payloadHash = sha256Of("").toHexString.toLower;
	string canonicalRequest = method ~ "\n" ~ canonicalURI ~ "\n" ~ canonicalQueryString ~ "\n" ~ canonicalHeadersString ~ "\n" ~ signedHeaders ~ "\n" ~ payloadHash;


	string algorithm = "AWS4-HMAC-SHA256";
	string credentialScope = curDateStr ~ "/" ~ region ~ "/" ~ service ~ "/" ~ "aws4_request";
	string stringToSign = algorithm ~ "\n" ~ xamztime ~ "\n" ~ credentialScope ~ "\n" ~ sha256Of(canonicalRequest).toHexString.toLower;

	auto signingKey = getSignatureKey(secretKey, curDateStr, region, service);

	string signature = hmac!SHA256(stringToSign.representation, signingKey).toHexString.toLower;

	string authorizationHeader = algorithm ~ " " ~ "Credential=" ~ accessKey ~ "/" ~ credentialScope ~ ", " ~ "SignedHeaders=" ~ signedHeaders ~ ", " ~ "Signature=" ~ signature;

	auto client = HTTP(endpoint ~ "?" ~ canonicalQueryString);
	client.method = HTTP.Method.get;
	client.addRequestHeader("x-amz-date", xamztime);
	client.addRequestHeader("Authorization", authorizationHeader);
	auto content = client.perform();

	writeln(content);
}