Lemmy Support

Hi all, apologies for the long first post, but I'm running around in circles trying to get my Lemmy instance up and running.

The instance runs, I can log in, but federation is just not working for me.

I can search for instances, they'll return the first 20 items like laid out here: https://join-lemmy.org/docs/en/administration/federation_getting_started.html but that's as far as I can get. If I click "subscribe" it'll sit on subscribe pending and the logs will show combinations of timeouts, server rejected, or unexpected EOF

lemmy-lemmy-1 | 2023-06-02T16:23:39.499641Z INFO HTTP request{http.method=GET http.scheme="http" http.host=<<myhost>> http.target=/api/v3/ws otel.kind="server" request_id=08d649ec-6662-4e8d-98bd-e48201f9f4bf http.status_code=101 otel.status_code="OK"}:send:send_lemmy_activity: activitypub_federation::core::activity_queue: Unable to connect to https://beehaw.org/inbox, aborting task <<myhost>>/activities/follow/1794b0b6-eefa-489e-b39f-41d05a217ac9: Request error: error sending request for url (https://beehaw.org/inbox): error trying to connect: unexpected EOF

INFO HTTP request{http.method=GET http.scheme="http" http.host=<<myhost>> http.target=/api/v3/ws otel.kind="server" request_id=08d649ec-6662-4e8d-98bd-e48201f9f4bf http.status_code=101 otel.status_code="OK"}:send:send_lemmy_activity: activitypub_federation::core::activity_queue: Unable to connect to https://beehaw.org/inbox, aborting task <<myhost>>/activities/undo/b3847def-0844-42ae-88c8-7c99aa9166ef: Request error: error sending request for url (https://beehaw.org/inbox): operation timed out

I have no idea what's happening honestly, and I'm tearing my hair out. I've read the docs, I think everything is turned on, https is working, blacklists and whitelists are emptied, I'm not sure what else to try.

For sanity here's my lemmy.ljson and docker-compose.

Any help is appreciated

{
  # for more info about the config, check out the documentation
  # https://join-lemmy.org/docs/en/administration/configuration.html
  # only few config options are covered in this example config

  setup: {
    # username for the admin user
    admin_username: "<<user>>"
    # password for the admin user
    admin_password: "<<password>>"
    # name of the site (can be changed later)
    site_name: "<<custom site name>>"
  }

  # the domain name of your instance (eg "lemmy.ml")
  hostname: "<<mydomain>>"
  # address where lemmy should listen for incoming requests
  bind: "0.0.0.0"
  # port where lemmy should listen for incoming requests
  port: 8536
  # Whether the site is available over TLS. Needs to be true for federation to work.
  tls_enabled: true

  # pictrs host
  pictrs: {
    url: "http://pictrs:8080/"
    # api_key: "API_KEY"
  }

  # settings related to the postgresql database
  database: {
    # name of the postgres database for lemmy
    database: "lemmy"
    # username to connect to postgres
    user: "lemmy"
    # password to connect to postgres
    password: "<<mydbpassword>>"
    # host where postgres is running
    host: "postgres"
    # port where postgres can be accessed
    port: 5432
    # maximum number of active sql connections
    pool_size: 5
  }
}

version: "3.3"

networks:
  # communication to web and clients
  lemmyexternalproxy:
  # communication between lemmy services
  lemmyinternal:
    driver: bridge
    internal: true

services:
  proxy:
    image: nginx:1-alpine
    networks:
      - lemmyinternal
      - lemmyexternalproxy
    ports:
      # only ports facing any connection from outside
      - 80:80 
      - 443:443
    volumes:
      - ./proxy/nginx.conf:/etc/nginx/nginx.conf:ro
      - ./proxy/proxy.conf:/etc/nginx/proxy.conf:ro
      - /cert:/cert:ro
      # setup your certbot and letsencrypt config 
    restart: always
    depends_on:
      - pictrs
      - lemmy-ui

  lemmy:
    image: dessalines/lemmy:0.17.3
    hostname: lemmy
    networks:
      #- lemmyexternalproxy
      - lemmyinternal
    restart: always
    environment:
      - RUST_LOG="debug,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug"
      #- RUST_LOG=verbose
      # - RUST_LOG="verbose,lemmy_server=info,lemmy_api=info,lemmy_api_common=info,lemmy_api_crud=info,lemmy_apub=info,lemmy_db_schema=info,lemmy_db_views=info,lemmy_db_views_actor=info,lemmy_db_views_moderator=info,lemmy_routes=info,lemmy_utils=info,lemmy_websocket=info"
    volumes:
      - ./lemmy.hjson:/config/config.hjson
    depends_on:
      - postgres
      - pictrs

  lemmy-ui:
    image: dessalines/lemmy-ui:0.17.3
    networks:
      - lemmyinternal
    environment:
      # this needs to match the hostname defined in the lemmy service
      - LEMMY_UI_LEMMY_INTERNAL_HOST=lemmy:8536
      # set the outside hostname here
      - LEMMY_UI_LEMMY_EXTERNAL_HOST=lemmy
      - LEMMY_HTTPS=true
    depends_on:
      - lemmy
    restart: always
    volumes:
      - ./volumes/lemmy-ui/extra_themes:/app/extra_themes

  pictrs:
    image: asonix/pictrs:0.3.1
    # this needs to match the pictrs url in lemmy.hjson
    hostname: pictrs
    # we can set options to pictrs like this, here we set max. image size and forced format for conversion
    # entrypoint: /sbin/tini -- /usr/local/bin/pict-rs -p /mnt -m 4 --image-format webp
    networks:
      - lemmyinternal
    environment:
      - PICTRS__API_KEY=API_KEY
    user: 991:991
    volumes:
      - ./volumes/pictrs:/mnt
    restart: always

  postgres:
    image: postgres:15-alpine
    # this needs to match the database host in lemmy.hson
    hostname: postgres
    networks:
      - lemmyinternal
    environment:
      - POSTGRES_USER=lemmy
      - POSTGRES_PASSWORD=<<mydbpassword>>
      - POSTGRES_DB=lemmy
    volumes:
      - ./volumes/postgres:/var/lib/postgresql/data
    restart: always