Flutter handle enter key for TextFormField, ignore focus change

I've got a dialog box in flutter with a TextFormField. I wanted to handle the user on the web hitting enter to accept the value in the field. The onFieldSubmitted() method works for this, but works actually TOO good. If they even take focus off the field, it triggers. I really want to only trigger for the Enter key. Do I have to use a RawKeyboardListener() for this? Or is there some way to know it wasn't an Enter key but a focus change that triggered the onFieldSubmitted()? Here is my example code snippet for the dialog. CustomTextField is just me extending the TextFormField to look a certain way each time.

  var res = await showDialog<String>(
    context: context,
    barrierDismissible: false,
    builder: (BuildContext context) {
      return AlertDialog(
        title: Row(
          children: <Widget>[
            Text('Add '),
            Container(
              width: 8,
            ),
            DropdownButton(
              value: _currentAddType,
              items: _addTypeDropDownMenuItems,
              onChanged: (value) => _changeAddType(value),
            ),
          ],
        ),
        shape: RoundedRectangleBorder(
            borderRadius: BorderRadius.all(Radius.circular(8.0))),
        content: SingleChildScrollView(
          child: ListBody(
            children: <Widget>[
              CustomTextField(
                controller: _roomNameController,
                textCapitalization: TextCapitalization.words,
                autoFocus: true,
                icon: Icon(Icons.text_fields),
                hint: "Room Name",
                onFieldSubmitted: (value) => { // <======== this is the problem, it works even if just losing focus
                  setState(() {
                    Navigator.of(context).pop('Add');
                  })
                },
              ),
              Container(
                height: 12,
              ),
              Text(
                'Select Gateway:',
                style: Theme.of(context).textTheme.headline6,
              ),
              Container(
                height: 4,
              ),
              DropdownButton(
                value: _currentGateway,
                items: _gatewayDropDownMenuItems,
                onChanged: (String gateway) {
                  _currentGateway = gateway;
                  setState(() {
                    Navigator.of(context)
                        .pop(); 
                  });
                  _addGatewayOrRoom();
                },
              ),
            ],
          ),
        ),
        actions: <Widget>[
          FlatButton(
            child: Text('Cancel'),
            onPressed: () {
              setState(() {
                Navigator.of(context).pop('Cancel');
              });
            },
          ),
          Container(
            width: 20,
          ),
          FlatButton(
            child: Text('Add'),
            onPressed: () {
              setState(() {
                Navigator.of(context).pop('Add');
              });
            },
          ),
        ],
      );
    },
  );
  if (res == 'Add') {
    await doAddRoom();
    _doRefresh();
  }