Hi All,

  Request your help to on how to improve the performance of the below code.

import std.conv: to;
import std.net.curl : get, HTTP, CurlOption;
import std.parallelism: parallel;
import std.range: chain, only;
import std.typecons: Tuple, tuple;

void main ()
{
 Array!(Tuple!(int,string)) apidata;
 Row[] result;
 string apihost = "abc.com"; int apiport = 1830; string apiuri = /getdata;
 string apiuser = "user"; string apipass = "pass";
 foreach(i, k; parallel(result,1))
    {
      string url = chain(apihost, only(':'), to!string(apiport), apiuri).to!string;
      string usrpass = chain(apiuser, only(':'), apipass).to!string;
      auto https = HTTP();
      https.handle.set(CurlOption.buffersize, 512000);
      https.handle.set(CurlOption.userpwd, usrpass);
      https.handle.set(CurlOption.connecttimeout, 600);
      https.handle.set(CurlOption.tcp_nodelay, 1);
      https.handle.set(CurlOption.http_version, 2);
      https.handle.set(CurlOption.sslversion,  1;
      https.handle.set(CurlOption.use_ssl,  3);
      https.handle.set(CurlOption.ssl_verifypeer, 0);
      https.handle.set(CurlOption.url, url);
      https.method(HTTP.Method.get);
      https.StatusLine st;
      https.onReceiveStatusLine = (https.StatusLine st) { if (st.code != 200) { throw new Exception(st.reason); } };
      ubyte[] content;
      https.onReceive = (ubyte[] data) { content ~= data; return data.length; };
      https.perform();
      scope(failure) { https.shutdown; exit(-4); } scope(exit) https.shutdown;
      apidata.insert(tuple(seq, cast(string) content));
 }
 return apidata[].sort;
}

From,
Vino.B

Just delete it

On Mon, Nov 9, 2020 at 9:00 PM Vino via Digitalmars-d-learn < digitalmars-d-learn@puremagic.com> wrote:

> Hi All,
>
>    Request your help to on how to improve the performance of the
> below code.
>
> import std.conv: to;
> import std.net.curl : get, HTTP, CurlOption;
> import std.parallelism: parallel;
> import std.range: chain, only;
> import std.typecons: Tuple, tuple;
>
> void main ()
> {
>   Array!(Tuple!(int,string)) apidata;
>   Row[] result;
>   string apihost = "abc.com"; int apiport = 1830; string apiuri =
> /getdata;
>   string apiuser = "user"; string apipass = "pass";
>   foreach(i, k; parallel(result,1))
>      {
>        string url = chain(apihost, only(':'), to!string(apiport),
> apiuri).to!string;
>        string usrpass = chain(apiuser, only(':'),
> apipass).to!string;
>        auto https = HTTP();
>        https.handle.set(CurlOption.buffersize, 512000);
>        https.handle.set(CurlOption.userpwd, usrpass);
>        https.handle.set(CurlOption.connecttimeout, 600);
>        https.handle.set(CurlOption.tcp_nodelay, 1);
>        https.handle.set(CurlOption.http_version, 2);
>        https.handle.set(CurlOption.sslversion,  1;
>        https.handle.set(CurlOption.use_ssl,  3);
>        https.handle.set(CurlOption.ssl_verifypeer, 0);
>        https.handle.set(CurlOption.url, url);
>        https.method(HTTP.Method.get);
>        https.StatusLine st;
>        https.onReceiveStatusLine = (https.StatusLine st) { if
> (st.code != 200) { throw new Exception(st.reason); } };
>        ubyte[] content;
>        https.onReceive = (ubyte[] data) { content ~= data; return
> data.length; };
>        https.perform();
>        scope(failure) { https.shutdown; exit(-4); } scope(exit)
> https.shutdown;
>        apidata.insert(tuple(seq, cast(string) content));
>   }
>   return apidata[].sort;
> }
>
> From,
> Vino.B
>

On Monday, 9 November 2020 at 19:55:07 UTC, Vino wrote:
> ...

The only curl option you need to set within the loop is the CurlOption.url. So your foreach block should look more like:

foreach (...) {
    string url = chain(apihost, only(':'), to!string(apiport), apiuri).to!string;
    https.handle.set(CurlOption.url, url);
    https.perform();
    scope(failure) exit(-4);
    scope(exit) https.shutdown;
    apidata.insert(tuple(seq, cast(string) content));
    content = [];
}

Every other line can be placed before the foreach.

On Monday, 9 November 2020 at 20:40:59 UTC, rinfz wrote:
> On Monday, 9 November 2020 at 19:55:07 UTC, Vino wrote:
>> ...
>
> The only curl option you need to set within the loop is the CurlOption.url. So your foreach block should look more like:
>
> foreach (...) {
>     string url = chain(apihost, only(':'), to!string(apiport), apiuri).to!string;
>     https.handle.set(CurlOption.url, url);
>     https.perform();
>     scope(failure) exit(-4);
>     scope(exit) https.shutdown;
>     apidata.insert(tuple(seq, cast(string) content));
>     content = [];
> }
>
> Every other line can be placed before the foreach.

In fact, you don't need url in there either since it's not dependent on loop variables, and you don't need the scope guards if this is running in main (you can move them out of the loop too).

foreach (...) {
    https.handle.set(CurlOption.url, url);
    https.perform();
    apidata.insert(tuple(seq, cast(string) content));
    content = [];
}

On Mon, Nov 9, 2020 at 9:50 PM rinfz via Digitalmars-d-learn < digitalmars-d-learn@puremagic.com> wrote:

> On Monday, 9 November 2020 at 20:40:59 UTC, rinfz wrote:
> > On Monday, 9 November 2020 at 19:55:07 UTC, Vino wrote:
> >> ...
> >
> > The only curl option you need to set within the loop is the CurlOption.url. So your foreach block should look more like:
> >
> > foreach (...) {
> >     string url = chain(apihost, only(':'), to!string(apiport),
> > apiuri).to!string;
> >     https.handle.set(CurlOption.url, url);
> >     https.perform();
> >     scope(failure) exit(-4);
> >     scope(exit) https.shutdown;
> >     apidata.insert(tuple(seq, cast(string) content));
> >     content = [];
> > }
> >
> > Every other line can be placed before the foreach.
>
> In fact, you don't need url in there either since it's not dependent on loop variables, and you don't need the scope guards if this is running in main (you can move them out of the loop too).
>
> foreach (...) {
>      https.handle.set(CurlOption.url, url);
>      https.perform();
>      apidata.insert(tuple(seq, cast(string) content));
>      content = [];
> }
>

In fact he does not need foreach. Because he use it on empty result

On Monday, 9 November 2020 at 20:57:33 UTC, Daniel Kozak wrote:
> On Mon, Nov 9, 2020 at 9:50 PM rinfz via Digitalmars-d-learn < digitalmars-d-learn@puremagic.com> wrote:
>
>> On Monday, 9 November 2020 at 20:40:59 UTC, rinfz wrote:
>> > On Monday, 9 November 2020 at 19:55:07 UTC, Vino wrote:
>> >> ...
>> >
>> > The only curl option you need to set within the loop is the CurlOption.url. So your foreach block should look more like:
>> >
>> > foreach (...) {
>> >     string url = chain(apihost, only(':'), to!string(apiport),
>> > apiuri).to!string;
>> >     https.handle.set(CurlOption.url, url);
>> >     https.perform();
>> >     scope(failure) exit(-4);
>> >     scope(exit) https.shutdown;
>> >     apidata.insert(tuple(seq, cast(string) content));
>> >     content = [];
>> > }
>> >
>> > Every other line can be placed before the foreach.
>>
>> In fact, you don't need url in there either since it's not dependent on loop variables, and you don't need the scope guards if this is running in main (you can move them out of the loop too).
>>
>> foreach (...) {
>>      https.handle.set(CurlOption.url, url);
>>      https.perform();
>>      apidata.insert(tuple(seq, cast(string) content));
>>      content = [];
>> }
>>
>
> In fact he does not need foreach. Because he use it on empty result

Hi All,

  The reason that the above code is within foreach loop is because there are several api's and each of the api's have separate configuration. These configuration are maintained in a table in MySQL database, the result contains the configuration details for each of these api's, below is the complete code.

Sequence
The code collects the data from several api's whose output is json.
The output of the api's are parsed, filtered, validated and stored in a array using another function.(using asdf json parser)
The data stores in array is then inserted into the table in MySQL database.(using hunt-database).

Code:
auto getData ()
{
 Array!(Tuple!(int,string)) apidata;
 Row[] result;
 result = getApiconf();      \\ fetch's the api configuration from the database table.
 foreach(i, k; parallel(result,1))
    {

      string apihost = result[i][0].get!(string);
      int apiport = result[i][1].get!(int);
      string apiuser = result[i][2].get!(string);
      string apipass = result[i][3].get!(string);
      int seq = result[i][4].get!int;
      string apiuri = result[i][5].get!(string);
      int connecttimeout = result[i][6].get!(int);
      int tcp_nodelay = result[i][7].get!(int);
      int http_version = result[i][8].get!(int);
      int sslversion = result[i][9].get!(int);
      int use_ssl = result[i][10].get!(int);
      int ssl_verifypeer = result[i][11].get!(int);

      string url = chain(apihost, only(':'), to!string(apiport), apiuri).to!string;
      string usrpass = chain(apiuser, only(':'), apipass).to!string;

      auto https = HTTP();
      https.handle.set(CurlOption.buffersize, 512000);
      https.handle.set(CurlOption.userpwd, usrpass);
      https.handle.set(CurlOption.connecttimeout, connecttimeout);
      https.handle.set(CurlOption.tcp_nodelay, tcp_nodelay);
      https.handle.set(CurlOption.http_version, http_version);
      https.handle.set(CurlOption.sslversion,  sslversion);
      https.handle.set(CurlOption.use_ssl,  use_ssl);
      https.handle.set(CurlOption.ssl_verifypeer, ssl_verifypeer);
      https.handle.set(CurlOption.url, url);
      https.method(HTTP.Method.get);
      https.StatusLine st;
      https.onReceiveStatusLine = (https.StatusLine st) { if (st.code != 200)
         {
           throw new Exception(st.reason);
         } };
      ubyte[] content;
      https.onReceive = (ubyte[] data) { content ~= data; return data.length; };
      https.perform();
      scope(failure) { https.shutdown; exit(-4); } scope(exit) https.shutdown;
      apidata.insert(tuple(seq, cast(string) content));
 }
 return apidata[].sort;
}

 From,
Vino.B