This causes a segfault when run with rdmd -gx:

void main()
{
    import std.experimental.allocator : allocatorObject, expandArray;
    import std.experimental.allocator.building_blocks.allocator_list : AllocatorList;
    import std.experimental.allocator.building_blocks.region : Region;
    import std.experimental.allocator.building_blocks.fallback_allocator : FallbackAllocator;
    import std.experimental.allocator.mallocator : Mallocator;

    {
        alias Alloc1 = FallbackAllocator!(
            AllocatorList!(n => Region!Mallocator(1024*1024)),
            Mallocator);
        auto alloc1 = allocatorObject(Alloc1());
        for (int ttt=0; ttt<10; ++ttt)
        {
            import std.stdio : writeln;
            writeln(ttt);
            import core.memory : GC;

            auto p = alloc1.allocate(60*1024);
            assert(p.length == 60*1024);
            auto p2 = alloc1.allocate(3*1024*1024);
            assert(p2.length == 3*1024*1024);
            GC.collect();
            alloc1.expandArray(p, 120*1024); // Segfault here
        }
    }
}


(Tested on DMD 2.094.2 and on https://run.dlang.io/is/p0FsOQ)

If the "GC.collect()" line is commented out, it works somehow.

Please help me understand why this is happening. This is a very reduced example of an issue I am facing.

Thank You,
Saurabh

On Thursday, 24 December 2020 at 16:12:31 UTC, Saurabh Das wrote:
> This causes a segfault when run with rdmd -gx:
[...]
On my machine its a "realloc(): invalid pointer".
From what does the allocator know, that `p' is somehow an array?

On Thursday, 24 December 2020 at 16:12:31 UTC, Saurabh Das wrote:
> This causes a segfault when run with rdmd -gx:
> *snip*

First, here's a reduced version:

void main() {
	import std.experimental.allocator: allocatorObject, expandArray;
	import std.experimental.allocator.building_blocks.allocator_list: AllocatorList;
	import std.experimental.allocator.building_blocks.region: Region;
	import std.experimental.allocator.building_blocks.fallback_allocator: FallbackAllocator;
	import std.experimental.allocator.mallocator: Mallocator;
	import core.memory: GC;
	import std.stdio;

	enum MB = 1024 * 1024;

	{
		alias Alloc1 = FallbackAllocator!(
				AllocatorList!(n => Region!Mallocator(MB)),
				Mallocator);
		auto alloc1 = allocatorObject(Alloc1());

		GC.collect;
		alloc1.allocate(MB);
	}
	
	writeln(5); // this never gets printed; segfault happens upon exiting the above scope
}


I'm not 100% sure where the segfault comes from--though I think it's a problem with AllocatorList--but as a workaround, try replacing ‘AllocatorList!(n => Region!Mallocator(MB))’ with ‘AllocatorList!(n => Region!Mallocator(MB), NullAllocator)’.

On Thursday, 24 December 2020 at 23:46:58 UTC, Elronnd wrote:
> reduced version:

Further reduction: Alloc1 can just be ‘AllocatorList!(n => Region!Mallocator(MB))’.

On Thursday, 24 December 2020 at 23:58:45 UTC, Elronnd wrote:
> On Thursday, 24 December 2020 at 23:46:58 UTC, Elronnd wrote:
>> reduced version:
>
> Further reduction: Alloc1 can just be ‘AllocatorList!(n => Region!Mallocator(MB))’.

Thank you for the reduced test case.

A small change to the test case seems to work in all the cases I've tested so far, maybe it can help diagnose the issue. If we use a pointer to construct the allocator, it seems to work fine:

void main() {
        import std.experimental.allocator: allocatorObject, expandArray;
        import std.experimental.allocator.building_blocks.allocator_list: AllocatorList;
        import std.experimental.allocator.building_blocks.region: Region;
        import std.experimental.allocator.building_blocks.fallback_allocator: FallbackAllocator;
        import std.experimental.allocator.mallocator: Mallocator;
        import core.memory: GC;
        import std.stdio;

        enum MB = 1024 * 1024;
        {
                alias Alloc1 = AllocatorList!(n => Region!Mallocator(MB));
                auto a1 = Alloc1();
                auto alloc1 = allocatorObject(&a1);

                GC.collect;
                alloc1.allocate(MB);
        }

        writeln(5); // this never gets printed; segfault happens upon exiting the above scope
}

Try to compile in debug mode, maybe you breach some contract.

On 24.12.20 17:12, Saurabh Das wrote:
> This causes a segfault when run with rdmd -gx:
> 
[...]
> 
> (Tested on DMD 2.094.2 and on https://run.dlang.io/is/p0FsOQ)
> 
> If the "GC.collect()" line is commented out, it works somehow.
> 
> Please help me understand why this is happening. This is a very reduced example of an issue I am facing.

Looks like a pretty nasty bug somewhere in std.experimental.allocator or (less likely) the GC. Further reduced code:

----
import core.memory: GC;
import core.stdc.stdlib: malloc;
import std.experimental.allocator: allocatorObject;
import std.experimental.allocator.building_blocks.allocator_list: AllocatorList;
import std.stdio: writeln;
import std.typecons: Ternary;

struct Mallocator
{
    int x = 42;
    void[] allocate(size_t n) nothrow @nogc
    {
        assert(n == 56); /* memory for bookkeeping, presumably */
        void* p = malloc(n);
        assert(p !is null);
        debug writeln(&this, " malloced ", p);
        return p[0 .. n];
    }
    Ternary owns(const void[] a) pure nothrow @nogc @safe
    {
        debug writeln(&this, " owns?    ", a.ptr);
        return a.ptr is null ? Ternary.no : Ternary.yes;
    }
    bool deallocateAll() pure nothrow @nogc @safe
    {
        assert(x == 42); /* fails; should pass */
        return true;
    }
    enum alignment = 1;
}

struct List
{
    AllocatorList!(n => Mallocator()) list;
    void[] allocate(size_t n) nothrow
    {
        return list.allocate(n);
    }
    bool deallocate(void[] a) pure nothrow @nogc @safe { return false; }
    enum alignment = 1;
}

void main()
{
    auto alloc1 = allocatorObject(List());
    () { ubyte[1000] stomp; } ();
    GC.collect();
    auto gca = new int;
}
----

Apparently, something calls deallocateAll on a Mallocator instance after the memory of that instance has been recycled by the GC. Maybe allocatorObject or AllocatorList keep a reference to GC memory out of sight of the GC.

On 26.12.20 13:59, ag0aep6g wrote:
> Looks like a pretty nasty bug somewhere in std.experimental.allocator or (less likely) the GC. Further reduced code:
> 
> ----
[...]
> ----
> 
> Apparently, something calls deallocateAll on a Mallocator instance after the memory of that instance has been recycled by the GC. Maybe allocatorObject or AllocatorList keep a reference to GC memory out of sight of the GC.

I've looked into it some more, and as far as I can tell this is what happens:

1) allocatorObject puts the AllocatorList instance into malloced memory.
2) The AllocatorList puts the Mallocator instance into GC memory, because its default BookkeepingAllocator is GCAllocator.
3) The GC recycles the memory of the Mallocator instance, because it's only reachable via the malloced AllocatorList instance and malloced memory isn't scanned by default.
4) Hell breaks loose because that recycled memory was not actually garbage.

I'm not so sure anymore if this qualifies as a bug in std.experimental.allocator. Maybe AllocatorList should be registering its GC allocations as roots?

As a solution/workaround, you can use NullAllocator for AllocatorList's BookkeepingAllocator:

----
import std.experimental.allocator.building_blocks.null_allocator :
    NullAllocator;
alias Alloc1 = FallbackAllocator!(
    AllocatorList!(n => Region!Mallocator(1024*1024), NullAllocator),
    Mallocator);
----

On Saturday, 26 December 2020 at 19:36:24 UTC, ag0aep6g wrote:
> On 26.12.20 13:59, ag0aep6g wrote:
>> Looks like a pretty nasty bug somewhere in std.experimental.allocator or (less likely) the GC. Further reduced code:
>> 
>> ----
> [...]
>> ----
>> 
>> Apparently, something calls deallocateAll on a Mallocator instance after the memory of that instance has been recycled by the GC. Maybe allocatorObject or AllocatorList keep a reference to GC memory out of sight of the GC.
>
> I've looked into it some more, and as far as I can tell this is what happens:
>
> 1) allocatorObject puts the AllocatorList instance into malloced memory.
> 2) The AllocatorList puts the Mallocator instance into GC memory, because its default BookkeepingAllocator is GCAllocator.
> 3) The GC recycles the memory of the Mallocator instance, because it's only reachable via the malloced AllocatorList instance and malloced memory isn't scanned by default.
> 4) Hell breaks loose because that recycled memory was not actually garbage.
>
> I'm not so sure anymore if this qualifies as a bug in std.experimental.allocator. Maybe AllocatorList should be registering its GC allocations as roots?
>
> As a solution/workaround, you can use NullAllocator for AllocatorList's BookkeepingAllocator:
>
> ----
> import std.experimental.allocator.building_blocks.null_allocator :
>     NullAllocator;
> alias Alloc1 = FallbackAllocator!(
>     AllocatorList!(n => Region!Mallocator(1024*1024), NullAllocator),
>     Mallocator);
> ----

Okay excellent. So there is a workaround atleast.

It also works with using Mallocator as the BookkeepingAllocator for AllocatorList.

I encountered a slightly differt seg fault too. I'm wondering whether it is related to this one:

import std.experimental.allocator: allocatorObject, expandArray;
import std.experimental.allocator.building_blocks.allocator_list: AllocatorList;
import std.experimental.allocator.building_blocks.region: Region;
import std.experimental.allocator.building_blocks.fallback_allocator: FallbackAllocator;
import std.experimental.allocator.mallocator: Mallocator;
import core.memory: GC;
import std.stdio;

void main()
{
    enum MB = 1024 * 1024;
    {
        alias Alloc1 = Region!Mallocator;
        auto a1 = Alloc1(MB);
        auto p1 = a1.allocate(10);
        auto a2 = a1;
        auto p2 = a2.allocate(10);

        writeln(a1.owns(p1));  // Prints Ternary.Yes - incorrect?
        writeln(a1.owns(p2));  // Prints Ternary.Yes - incorrect?
        writeln(a2.owns(p1));  // Prints Ternary.Yes
        writeln(a2.owns(p2));  // Prints Ternary.Yes

        writeln(4); // This is printed
    }

    writeln(5); // this never gets printed; segfault happens upon exiting the above scope
}