Remove Group of Permissions in Django

I have list of users where the admin can add/remove group of permissions for them. When I want to add everything works fine but when removing it gives me a success method but nothing is removed.

def add_remove_role(role_id, user_id, is_checked):
'''
add/remove role for a user
@param role_id: id of role to be removed or added
@param user_id: id of user that should add or remove for him a role
@param is_checked: if true then add role else remove role
'''
try:
    user = User.objects.filter(id=user_id).first()
    role = Group.objects.get(id=role_id) 

    role.user_set.add(user) if is_checked else role.user_set.remove(user)
    
    log.debug('successfully added/removed the role "%s" for user "%s" ' %(role_id, user_id))
    return True

except Exception as e:
    log.error('an error has occurred while adding/removing role of id "%s" for user "%s" ' %(role_id, user_id))
    return False