FIX: Double cosets in niceomorph group

This fixes #5539
gap-system · Dec 8, 2023 · 6241704 · 6241704
1 parent c6d3db2
commit 6241704
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 1 deletion.
diff --git a/lib/csetgrp.gi b/lib/csetgrp.gi
@@ -1422,7 +1422,9 @@ local c, flip, maxidx, cano, tryfct, p, r, t,
         # in the normal case, we can obtain the other orbits easily via
         # the orbit theorem (same stabilizer)
         if Size(lst)/Size(st)<10 then
-          rt:=Orbit(lst,One(st),
+          # if the group `st` is handled by a nice monomorphism, the
+          # identity might not be the canonical element for the subgroup.
+          rt:=Orbit(lst,CanonicalRightCosetElement(st,One(st)),
             function(rep,g) return CanonicalRightCosetElement(st,rep*g);end);
         else
           rt:=RightTransversal(lst,st:noascendingchain);

diff --git a/tst/testbugfix/2023-12-07-DCoset.tst b/tst/testbugfix/2023-12-07-DCoset.tst
@@ -0,0 +1,5 @@
+# Issue #5539
+gap> T := SmallGroup( 4, 2 );;
+gap> G := SmallGroup( 9, 2 );;
+gap> A := AutomorphismGroup(T);;
+gap> homs := AllHomomorphisms(G,A);;