Django - Exclude options from ModelChoiceField

I am having a hard time trying to exclude choices from a ModelChoiceField using queryset. I am not very experienced with django and python, your help would be very appreciated.

I am trying to make a form to assign users to a project. I use an inlineformset and I'd like to remove the users that are already assigned to this project from the user combobox in the formset.

The ModelForm:

class ProjectUserForm(forms.ModelForm):
    user = forms.ModelChoiceField(queryset=User.objects.all())

    def __init__(self, *args, **kwargs):
        project = kwargs.pop('project', None)
        super(ProjectUserForm, self).__init__(*args, **kwargs)

        if project:
            self.fields['user'].queryset = User.objects.exclude(project=project)

    class Meta:
    model = ProjectUser

The View:

@login_required
def project_update(request, pk):
    if pk is None:
        project = Project()
    else:
        project = Project.objects.get(id=pk)

    ProjectUserFormset = inlineformset_factory(Project, ProjectUser,
        form=ProjectUserForm(project=project),
        extra=0,
        max_num=User.objects.count(),
        fk_name='project',
        can_delete=False,
    )

    if request.method == 'POST':
        project_form = ProjectForm(
            request.POST,
            instance=project
        )
        project_user_formset = ProjectUserFormset(
            request.POST,
            instance=project,
            prefix='project_user'
        )  
        if project_form.is_valid() and project_user_formset.is_valid():
            project = project_form.save(commit=False)
            project_user_formset.save()
            project.save()

    else:
        project_form = ProjectForm(instance=project)
        project_user_formset = ProjectUserFormset(
            instance=project,
            prefix='project_user'
        )

    return render(request, "Nexquality/project_form.html", {
        'title': 'Modify project: '+project.name,
        'form': project_form,
        'project_user_formset': project_user_formset,
    })